diff --git a/tiera-rerun-fixed-moonep/STATUS.txt b/tiera-rerun-fixed-moonep/STATUS.txt index 5071a99f9d8704c523b17d1623e379cf2d36d064..08443f29a00d721b9bee78cf75277151183122ea 100644 --- a/tiera-rerun-fixed-moonep/STATUS.txt +++ b/tiera-rerun-fixed-moonep/STATUS.txt @@ -1,9 +1,9 @@ study_root: /workspace/tiera-rerun-fixed-moonep gbs: 16 -updated: 2026-08-30T04:52:38+00:00 -current: echo__tierbprobe__gbs16 +updated: 2026-08-30T05:06:43+00:00 +current: ultraep__tierbprobe__gbs16 completed steps: failed steps: - (none) + FAIL echo tierbprobe 16 2026-08-30T04:52:38+00:00 2026-08-30T05:06:38+00:00 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe diff --git a/tiera-rerun-fixed-moonep/hf_watcher.log b/tiera-rerun-fixed-moonep/hf_watcher.log index 1b9c2665da85258862098e967b09dd7cbd71e488..5fd840368bd9bf338d7f61c122eeafa24fbdc1f7 100644 --- a/tiera-rerun-fixed-moonep/hf_watcher.log +++ b/tiera-rerun-fixed-moonep/hf_watcher.log @@ -4,3 +4,6 @@ Watching /workspace/tiera-rerun-fixed-moonep -> HF every 600s (kill to stop) uploaded: /workspace/tiera-rerun-fixed-moonep -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tiera-rerun-fixed-moonep https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main [2026-08-30T05:02:40+00:00] backup pass + uploaded: /workspace/tiera-rerun-fixed-moonep -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tiera-rerun-fixed-moonep +https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main +[2026-08-30T05:12:45+00:00] backup pass diff --git a/tiera-rerun-fixed-moonep/steps.tsv b/tiera-rerun-fixed-moonep/steps.tsv index f0e9f9ae8b797197bee471d47bb17cd69bd82498..a2286e7e78edea7946c3bcf2710c359fd2877ace 100644 --- a/tiera-rerun-fixed-moonep/steps.tsv +++ b/tiera-rerun-fixed-moonep/steps.tsv @@ -1 +1,2 @@ status campaign phase gbs started ended result_dir +FAIL echo tierbprobe 16 2026-08-30T04:52:38+00:00 2026-08-30T05:06:38+00:00 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe diff --git a/tiera-rerun-fixed-moonep/study.log b/tiera-rerun-fixed-moonep/study.log index 6ffa67840555ca52d3d1c0c73708f9bdcfaa701c..7323a2d5d231fb8f8533e2a0460616bca311fe17 100644 --- a/tiera-rerun-fixed-moonep/study.log +++ b/tiera-rerun-fixed-moonep/study.log @@ -52,3 +52,140 @@ driver_repo: /workspace/blackwell_run_scripts @ 774e9e2 [guard] iter 12/12 (80s) -------------------------------------------------------------------------- [05:02:24] CELL G4__echo__matched__seq4096__gbs16__cap2p00 (slots=16) + [guard] iter 4/12 (70s) + [guard] iter 12/12 (80s) + [guard] tearing down process group (outcome=DONE) +[05:03:48] G4 echo: measured required=1.0166015625 at C=2.00 -> validating C=1.068359375 (safety=+5%) +-------------------------------------------------------------------------- +[05:03:48] CELL G4__echo__matched__seq4096__gbs16__cap1p068359375 (slots=16) + [guard] iter 4/12 (70s) + [guard] iter 12/12 (80s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[05:05:12] CELL G4__echo__matched__seq4096__gbs16__cap1p068359375__forcebal (slots=16) + [guard] iter 7/12 (70s) + [guard] iter 12/12 (80s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- + +============================================================== + SWEEP COMPLETE ok=10 failed=0 +============================================================== +config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir +G1 echo matched 1 4096 16 2.00 none 1 OK 671.53 666.90 125477 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G1__echo__matched__seq4096__gbs16__cap2p00/rep1 +G2 echo matched 4 4096 16 2.00 none 1 OK 675.77 669.20 130289 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap2p00/rep1 +G2 echo matched 4 4096 16 1.08203125 none 1 OK 660.43 642.00 119267 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125/rep1 +G2 echo matched 4 4096 16 1.08203125 none+forcebal 1 OK 628.07 616.10 119267 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G2__echo__matched__seq4096__gbs16__cap1p08203125__forcebal/rep1 +G3 echo matched 8 4096 16 2.00 none 1 OK 721.76 713.60 136111 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap2p00/rep1 +G3 echo matched 8 4096 16 1.072265625 none 1 OK 681.37 670.20 122853 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625/rep1 +G3 echo matched 8 4096 16 1.072265625 none+forcebal 1 OK 664.27 659.10 122899 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G3__echo__matched__seq4096__gbs16__cap1p072265625__forcebal/rep1 +G4 echo matched 16 4096 16 2.00 none 1 OK 829.14 790.80 146037 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap2p00/rep1 +G4 echo matched 16 4096 16 1.068359375 none 1 OK 775.73 738.90 129621 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375/rep1 +G4 echo matched 16 4096 16 1.068359375 none+forcebal 1 OK 768.04 733.40 129579 /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/G4__echo__matched__seq4096__gbs16__cap1p068359375__forcebal/rep1 + + Results: /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/results.tsv + Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16 +============================================================== +10 probe runs -> /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe/tierbprobe_gbs16/probe_summary.tsv +run arm cap rawExpI preRankI postRankI drop% reqCap +-------------------------------------------------------------------------------------------------------------------- +G1__echo__matched__seq4096__gbs16__cap2p00/rep1 natural 2.00 3.6198* 3.6198 1.0827 1.9062 +G2__echo__matched__seq4096__gbs16__cap1p08203125/rep natural 1.08203125 3.8720* 2.1703 1.0000 1.0273 +G2__echo__matched__seq4096__gbs16__cap1p08203125__fo forced 1.08203125 1.0201 1.0069 1.0000 1.0234 +G2__echo__matched__seq4096__gbs16__cap2p00/rep1 natural 2.00 3.8724* 2.1875 1.0000 1.0273 +G3__echo__matched__seq4096__gbs16__cap1p072265625/re natural 1.072265625 3.9096* 1.7838 1.0000 1.0195 +G3__echo__matched__seq4096__gbs16__cap1p072265625__f forced 1.072265625 1.0231 1.0048 1.0000 1.0176 +G3__echo__matched__seq4096__gbs16__cap2p00/rep1 natural 2.00 3.9096* 1.7916 1.0000 1.0195 +G4__echo__matched__seq4096__gbs16__cap1p068359375/re natural 1.068359375 3.9303* 1.5507 1.0000 1.0166 +G4__echo__matched__seq4096__gbs16__cap1p068359375__f forced 1.068359375 1.0249 1.0034 1.0000 1.0137 +G4__echo__matched__seq4096__gbs16__cap2p00/rep1 natural 2.00 3.9301* 1.5543 1.0000 1.0166 + + * At least one globally aggregated snapshot reached the E/K expert-Imax ceiling. + Inspect raw_expert_imax_mean and hottest_expert_token_fraction_mean before generalising. + +ECHO note: echo_logged_pre_rank_metric_valid=0 and echo_logged_load_reduction_valid=0 are intentional. +The branch's built-in fields mix source-rank and destination-rank axes; this tool reconstructs both globally. + + TIER B PROBE -- what these logs contain, per backend + ---------------------------------------------------- + Capacity matrix: + ECHO: start C=2.00; on OOM descend 1.75, 1.50, 1.25. Stop at + the highest fitting zero-drop capacity. If a fitting capacity drops, + smaller values cannot help and the model is blocked. + UltraEP: measure required_capacity at C=1.25, round it UP to the next + 0.01, then validate that capacity once. + MoonEP: natural + forced, no capacity factor. + + The forced arm runs once at the selected/high-headroom capacity. It validates + instrumentation and is not part of capacity selection. + + Each backend instruments a different thing: + + echo local raw-router histograms and local post-reroute placement. + reduce_probe.py aligns and sums all eight source ranks before + calculating raw_expert_*, pre_home_rank_*, + post_physical_rank_* and valid_max_rank_reduction_*. + + ultraep global pre/post rank vectors plus explicit drop accounting: + drop_rate_pct and required_capacity -- the capacity factor the run + WOULD have needed. The vectors are duplicated identically on all + logging ranks and are deduplicated, not summed. + + moonep local raw-router histograms, actual payload rows per physical + destination, and the bytes-copy side. The reducer globally sums + each (iteration, layer) snapshot first. ECHO reports no comparable + byte counters, so MoonEP bytes are a self-comparison across models, + never a cross-backend ranking. + + Do NOT quote ECHO's logged pre_echo imbalance_ratio or + max_load_reduction_pct. The branch mixes source-rank and destination-rank + axes; the reducer marks both invalid and reconstructs the comparison globally. + + Two boundary cases: + raw_expert_imax_max == E/K + -> at least one global snapshot had one expert in every + token's top-K set. Inspect the MEAN before generalising. + required_capacity or min-safe-capacity at the configured capacity + -> clipped by the buffer; it means "at or over", not + "fits comfortably". + + tools/reduce_probe.py has already written probe_summary.tsv and .json here. + + Campaign results: /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe + Roll up: python3 /workspace/blackwell_run_scripts/granularity/tools/collect.py /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe --compare +Already logged in to Hugging Face as: AbhishekGhosh1998 + uploaded: /workspace/tiera-rerun-fixed-moonep/echo/tierbprobe -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tiera-rerun-fixed-moonep/echo/tierbprobe +https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main + +=== STEP ultraep__tierbprobe__gbs16 +============================================================== + CAMPAIGN: ultraep backend=ultraep phase=tierbprobe + Megatron: /workspace/Megatron-UltraEP @ ultraep-token-drop-analysis + GBS=16 seq=4096 repeats=3 nsys=1 + results -> /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe +============================================================== +-------------------------------------------------------------------------- +[05:06:59] CELL G1__ultraep__matched__seq4096__gbs16__cap1p25 (slots=1) + [guard] iter 4/12 (60s) + [guard] iter 12/12 (70s) + [guard] tearing down process group (outcome=DONE) +[05:08:13] G1 ultraep: validating C=1.8125 (safety=+5%) +-------------------------------------------------------------------------- +[05:08:13] CELL G1__ultraep__matched__seq4096__gbs16__cap1p8125 (slots=1) + [guard] iter 4/12 (60s) + [guard] iter 12/12 (70s) + [guard] tearing down process group (outcome=DONE) +[05:09:28] G1 ultraep: C=1.8125 dropped; increasing headroom +-------------------------------------------------------------------------- +[05:09:28] CELL G2__ultraep__matched__seq4096__gbs16__cap1p25 (slots=4) + [guard] iter 4/12 (60s) + [guard] iter 12/12 (70s) + [guard] tearing down process group (outcome=DONE) +[05:10:42] G2 ultraep: validating C=1.0625 (safety=+5%) +-------------------------------------------------------------------------- +[05:10:42] CELL G2__ultraep__matched__seq4096__gbs16__cap1p0625 (slots=4) + [guard] iter 5/12 (60s) + [guard] iter 12/12 (70s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[05:11:57] CELL G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal (slots=4) diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/manifest.txt b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/manifest.txt new file mode 100644 index 0000000000000000000000000000000000000000..2193f8828a815dfe9c9dce1e15f98ec0dc8e3fcd --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/manifest.txt @@ -0,0 +1,19 @@ +timestamp: 2026-08-30T05:06:58+00:00 +host: sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0 +gpu: NVIDIA B200 x8 +driver: 580.95.05 +study_scripts_branch: granularity-ultraep +study_scripts_commit: 818ab62 +campaign: ultraep +backend: ultraep +phase: tierbprobe +gbs: 16 +seq: 4096 +configs: G1 G2 G3 G4 +capacity: 1.25 +repeats: 3 +nsys: 1 +megatron_repo: /workspace/Megatron-UltraEP +megatron_branch_required: ultraep-token-drop-analysis +megatron_branch_actual: ultraep-token-drop-analysis +megatron_commit: eaf829e7 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/pip_freeze.txt b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/pip_freeze.txt new file mode 100644 index 0000000000000000000000000000000000000000..2911a706106f0055210d5f3631cc878b909ba134 --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/pip_freeze.txt @@ -0,0 +1,317 @@ +absl-py==2.3.0 +aiohappyeyeballs @ file:///rapids/aiohappyeyeballs-2.6.1-py3-none-any.whl#sha256=f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 +aiohttp @ file:///rapids/aiohttp-3.12.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=94f98e0e5a49f89b252e115844f756c04fc8050f38252a32a3dd994ce8121f10 +aiosignal @ file:///rapids/aiosignal-1.3.2-py2.py3-none-any.whl#sha256=45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5 +annotated-types==0.7.0 +anyio==4.9.0 +apex @ file:///opt/pytorch/apex/dist/apex-0.1-cp312-cp312-linux_x86_64.whl#sha256=a1508e4a386362e9c8f46c92ccada9110e421212b964c694235d0fd5a224b170 +argon2-cffi==25.1.0 +argon2-cffi-bindings==21.2.0 +arrow==1.3.0 +asciitree @ file:///rapids/asciitree-0.3.3-py3-none-any.whl#sha256=3e507ccfaeb417823158091fdb9d26b0205f6a59ba680b479a71db65c73d01d4 +asttokens==3.0.0 +astunparse==1.6.3 +async-lru==2.0.5 +attrs==25.3.0 +audioread==3.0.1 +babel==2.17.0 +beautifulsoup4==4.13.4 +black==25.1.0 +bleach==6.2.0 +blis==0.7.11 +cachetools @ file:///rapids/cachetools-6.0.0-py3-none-any.whl#sha256=82e73ba88f7b30228b5507dce1a1f878498fc669d972aef2dde4f3a3c24f103e +catalogue==2.0.10 +certifi==2025.4.26 +cffi==1.17.1 +charset-normalizer==3.4.2 +click==8.5.0 +cloudpathlib==0.21.1 +cloudpickle @ file:///rapids/cloudpickle-3.1.1-py3-none-any.whl#sha256=c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e +cmake==3.31.6 +comm==0.2.2 +confection==0.1.5 +contourpy==1.3.2 +cuda-bindings @ file:///rapids/cuda_bindings-12.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=ff0e28d1e34758654b9c961e1f55e4786e49aee6a4dbceaf3cc24c46c672df7e +cuda-python @ file:///rapids/cuda_python-12.9.0-py3-none-any.whl#sha256=926acba49b2c0a0374c61b7c98f337c085199cf51cdfe4d6423c4129c20547a7 +cudf @ file:///rapids/cudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b779a8e8c0e8577b9b560895958496da211bfac49cdc4f11a49e8d18cb82e348 +cudf-polars @ file:///rapids/cudf_polars-25.4.0-py3-none-any.whl#sha256=227b6c5147e93ea86282c8f7219118f916769fa2ca0cdd931e97a148aa66b869 +cugraph @ file:///rapids/cugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b837c2dc2f7b235c702e4f8df64479b8fcbb342f848516d5faf0cdc4b5f31f29 +cugraph-service-client @ file:///rapids/cugraph_service_client-25.4.0-py3-none-any.whl#sha256=7e568c3e41154b9752f5173555ffcc282ac52d72343feeb79653b2e2b1232470 +cugraph-service-server @ file:///rapids/cugraph_service_server-25.4.0-py3-none-any.whl#sha256=a2830dcffafcccb16e12ebf85b98f8f247869d38c01ab4dd8664a1e7439d31be +cuml @ file:///rapids/cuml-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1e90a464a18b094e24797f1676688c5d7696c4cf97c934504214a53d5aa09aa5 +cupy-cuda12x @ file:///rapids/cupy_cuda12x-13.3.0-cp312-cp312-linux_x86_64.whl#sha256=6a2dfa0aa1e8189a10d1534276c08f1d2f73b747fd28bfbe7b9f4ef66d9af73f +cuvs @ file:///rapids/cuvs-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b12f6b9f80e053a3b38542c2947d2516e468c02cad54a13d8b1a4a61d5aa7914 +cycler==0.12.1 +cymem==2.0.11 +Cython==3.1.1 +dask @ file:///rapids/dask-2025.2.0-py3-none-any.whl#sha256=f0fdeef6ceb0a06569d456c9e704f220f7f54e80f3a6ea42ab98cea6bc642b6e +dask-cuda @ file:///rapids/dask_cuda-25.4.0-py3-none-any.whl#sha256=442d87da0a95d00ca2a262b64c76e785a32ad905c82000fa2227932f81390e19 +dask-cudf @ file:///rapids/dask_cudf-25.4.0-py3-none-any.whl#sha256=5e7cea0b26daea17da6e0bb457435e24460f76ba740358a3a18e239d6015c95c +datasets==5.0.1 +debugpy==1.8.14 +decorator==5.2.1 +-e git+https://github.com/deepseek-ai/DeepEP.git@34152ae28f80bcc3ee38d7a12cb2ad87cfd4ea72#egg=deep_ep +defusedxml==0.7.1 +dill==0.4.1 +distributed @ file:///rapids/distributed-2025.2.0-py3-none-any.whl#sha256=368462084be47092d0a835f6c64c9b852da64dc88302b76d5f93bd5131228c67 +distributed-ucxx @ file:///rapids/distributed_ucxx-0.43.0-py3-none-any.whl#sha256=94f60d0ccb0c94ba8dbfa1eaf455874d18aaab17e2ae218c38590e9fdc2d721d +dm-tree==0.1.9 +einops==0.8.1 +execnet==2.1.1 +executing==2.2.0 +expecttest==0.3.0 +fasteners @ file:///rapids/fasteners-0.19-py3-none-any.whl#sha256=758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237 +fastjsonschema==2.21.1 +fastrlock @ file:///rapids/fastrlock-0.8.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl#sha256=5f13ec08f1adb1aa916c384b05ecb7dbebb8df9ea81abd045f60941c6283a670 +filelock==3.18.0 +flash_attn @ file:///opt/pytorch/flash_attn-2.7.4.post1-cp312-cp312-linux_x86_64.whl#sha256=a10455d6c56b387785b7413fd0bbcb392b42b3d4f4b88fda5d86cc8a9ce9182a +fonttools==4.58.1 +fqdn==1.5.1 +frozenlist @ file:///rapids/frozenlist-1.6.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=4e1be9111cb6756868ac242b3c2bd1f09d9aea09846e4f5c23715e7afb647103 +fsspec==2025.5.1 +gast==0.6.0 +grpcio==1.62.1 +h11==0.16.0 +hf-xet==1.6.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.29.0 +hypothesis==6.130.8 +idna==3.10 +importlib_metadata==8.7.0 +iniconfig==2.1.0 +intel-openmp==2021.4.0 +ipykernel==6.29.5 +ipython==9.3.0 +ipython_pygments_lexers==1.1.1 +isoduration==20.11.0 +isort==6.0.1 +jedi==0.19.2 +Jinja2==3.1.6 +joblib==1.5.1 +json5==0.12.0 +jsonpointer==3.0.0 +jsonschema==4.24.0 +jsonschema-specifications==2025.4.1 +jupyter-events==0.12.0 +jupyter-lsp==2.2.5 +jupyter_client==8.6.3 +jupyter_core==5.8.1 +jupyter_server==2.16.0 +jupyter_server_terminals==0.5.3 +jupyterlab==4.4.3 +jupyterlab_code_formatter==3.0.2 +jupyterlab_pygments==0.3.0 +jupyterlab_server==2.27.3 +jupyterlab_tensorboard_pro @ file:///builder/jupyterlab_tensorboard_pro-4.0.0-py2.py3-none-any.whl#sha256=6bfcb5ced3358eb72150f4b2ab91312faef2f0ca14d845e8379d8fa52df6902e +jupytext==1.17.2 +kiwisolver==1.4.8 +kvikio @ file:///rapids/kvikio-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=8b40fd8ea9e4a2974c04b50b7fc26b06094ac807f9a233281d0bafbaa71b1d85 +langcodes==3.5.0 +language_data==1.3.0 +lazy_loader==0.4 +libcudf @ file:///rapids/libcudf-25.4.0-py3-none-linux_x86_64.whl#sha256=238320b4958c1737bba2a609673e2539a78216b811b5498a68e00395d9b021a4 +libcugraph @ file:///rapids/libcugraph-25.4.0-py3-none-linux_x86_64.whl#sha256=29085e901b2f7569ff2653bb1355fe6fad20f2fcaa585f5649da9867d8df766d +libcuml @ file:///rapids/libcuml-25.4.0-py3-none-linux_x86_64.whl#sha256=8e2e444c72b0e995e15a48da37ff12fb410a47a2baa69ba86ccc9b1a3db91e8c +libcuvs @ file:///rapids/libcuvs-25.4.0-py3-none-linux_x86_64.whl#sha256=e2f8db61c92b769c590dd752f43439ddd62b2f6592a3c337362b6c1ef220c3b9 +libkvikio @ file:///rapids/libkvikio-25.4.0-py3-none-linux_x86_64.whl#sha256=da8a5713b05fc478bf40529a3e0fb61a2d605bb5a70ca4d461f96a6005dbe8b7 +libraft @ file:///rapids/libraft-25.4.0-py3-none-linux_x86_64.whl#sha256=b921dec005aa4a321dd156776a18376feb47c96a5b67eec125c8a8e19edd1e24 +librmm @ file:///rapids/librmm-25.4.0-py3-none-linux_x86_64.whl#sha256=ff951811bda729d4fa1109ab906225ae1f489a6ec2eeab5d1b674cc34bd4422f +librmm-cu12 @ file:///rapids/librmm_cu12-25.4.0-py3-none-any.whl#sha256=29296ef2b0e90679a0db280e86015be12d6ab9f4e62a181aa085f62dbadd06b3 +librosa==0.11.0 +libucxx @ file:///rapids/libucxx-0.43.0-py3-none-linux_x86_64.whl#sha256=f15849d7ac2fb68f0fd8998838a13321ddcfb0c594f7e6cbc401a3bcc4750566 +lightning-thunder @ file:///opt/pytorch/lightning-thunder +lightning-utilities==0.14.3 +lintrunner==0.12.7 +llvmlite==0.42.0 +locket @ file:///rapids/locket-1.0.0-py2.py3-none-any.whl#sha256=b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3 +looseversion==1.3.0 +marisa-trie==1.2.1 +Markdown==3.8 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +matplotlib==3.10.3 +matplotlib-inline==0.1.7 +mdit-py-plugins==0.4.2 +mdurl==0.1.2 +-e git+ssh://git@github.com/sriram-17-17/my-Megatron-MoE-Echo.git@b099fd8494b09e8c08ad34e26abf91fb6c83636d#egg=megatron_core +mistune==3.1.3 +mkl==2021.1.1 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +ml_dtypes==0.6.0 +mock==5.2.0 +-e git+ssh://git@github.com/sriram-17-17/MoonEP.git@d8bc6667fad7177e6e09b637d773fa67f08bae93#egg=moonep +mpmath==1.3.0 +msgpack==1.1.0 +multidict @ file:///rapids/multidict-6.4.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=a920f9cf2abdf6e493c519492d892c362007f113c94da4c239ae88429835bad1 +multiprocess==0.70.19 +murmurhash==1.0.13 +mypy_extensions==1.1.0 +nbclient==0.10.2 +nbconvert==7.16.6 +nbformat==5.10.4 +nest-asyncio==1.6.0 +networkx==3.5 +ninja==1.11.1.4 +notebook==7.4.3 +notebook_shim==0.2.4 +numba==0.59.1 +numba-cuda @ file:///rapids/numba_cuda-0.4.0-py3-none-any.whl#sha256=5b95a5f42068771e37142944d44fe844fa87f1b2f89051461a4cb0af91bcc28a +numcodecs @ file:///rapids/numcodecs-0.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=da2230484e6102e5fa3cc1a5dd37ca1f92dfbd183d91662074d6f7574e3e8f53 +numpy==1.26.4 +nvdlfw_inspect @ git+https://github.com/NVIDIA/nvidia-dlfw-inspect.git@6b60eb0e675606fb2fbbfcb12667ecbec75eaf87 +nvfuser==0.2.27a0+9bf5aca +nvidia-cuda-cccl-cu12==12.9.27 +nvidia-cudnn-frontend @ file:///opt/pytorch/lightning-thunder/tmp/cudnn_frontend +nvidia-cutlass-dsl==4.4.2 +nvidia-cutlass-dsl-libs-base==4.4.2 +nvidia-dali-cuda120==1.50.0 +nvidia-ml-py @ file:///rapids/nvidia_ml_py-12.575.51-py3-none-any.whl#sha256=eb8641800d98ce40a22f479873f34b482e214a7e80349c63be51c3919845446e +nvidia-modelopt==0.29.0 +nvidia-modelopt-core==0.29.0 +nvidia-nvcomp-cu12==4.2.0.14 +nvidia-nvimgcodec-cu12==0.5.0.13 +nvidia-nvjpeg-cu12==12.4.0.16 +nvidia-nvjpeg2k-cu12==0.8.1.40 +nvidia-nvshmem-cu12==3.7.2 +nvidia-nvtiff-cu12==0.5.0.67 +nvidia-resiliency-ext==0.4.0 +nvtx @ file:///rapids/nvtx-0.2.11-cp312-cp312-linux_x86_64.whl#sha256=73521e37cba42b4db0b3cd68fc5eb814e55c500a61eeed6edcdf6746315ac53e +nx-cugraph @ file:///rapids/nx_cugraph-25.4.0-py3-none-any.whl#sha256=e625444486cc899a77bd48e7da76a2edd95c0006dd95fbf323745feae7f33111 +onnx @ file:///opt/pytorch/pytorch/third_party/onnx +onnx-ir==1.0.0 +onnxscript==0.7.1 +opt_einsum==3.4.0 +optree==0.16.0 +overrides==7.7.0 +packaging==23.2 +pandas @ file:///rapids/pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319 +pandocfilters==1.5.1 +parso==0.8.4 +partd @ file:///rapids/partd-1.4.2-py3-none-any.whl#sha256=978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f +pathspec==0.12.1 +pexpect==4.9.0 +pillow==11.2.1 +platformdirs==4.3.8 +pluggy==1.6.0 +ply @ file:///rapids/ply-3.11-py2.py3-none-any.whl#sha256=096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce +polars @ file:///rapids/polars-1.25.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=f7fcbb4f476784384ccda48757fca4e8c2e2c5a0a3aef3717aaf56aee4e30e09 +polygraphy==0.49.20 +pooch==1.8.2 +preshed==3.0.10 +prometheus_client==0.22.1 +prompt_toolkit==3.0.51 +propcache @ file:///rapids/propcache-0.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=563f9d8c03ad645597b8d010ef4e9eab359faeb11a0a2ac9f7b4bc8c28ebef25 +protobuf==4.24.4 +psutil==7.0.0 +ptyprocess==0.7.0 +PuLP==3.2.1 +pure_eval==0.2.3 +pyarrow==25.0.1 +pybind11==2.13.6 +pybind11_global==2.13.6 +pycocotools @ git+https://github.com/nvidia/cocoapi.git@08350a157b41f0755a536441b5abec89ab934c5b#subdirectory=PythonAPI +pycparser==2.22 +pydantic==2.11.5 +pydantic_core==2.33.2 +Pygments==2.19.1 +pylibcudf @ file:///rapids/pylibcudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=401b8bfd022d06f84e120779ca7c3d01a28b68762aaa8f7873b10d6b1975cf6a +pylibcugraph @ file:///rapids/pylibcugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=061c7389a88c815b54fd5d3a3584fb87bf693e5602f5be335260d0ba575ba557 +pylibcugraphops @ file:///rapids/pylibcugraphops-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=a25ef4b22166c80718d5c7e5bddb755e81a2fe340fa5becd0ac36ec3edfb208a +pylibraft @ file:///rapids/pylibraft-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=f2ecd23fe51f9c2caa87bbbe85cb32fcaff4ecfb52ceb9a9991bc50f82782c2e +pylibwholegraph @ file:///rapids/pylibwholegraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b82913d1e38bda76d8c272fbd8cc9072d987209d3d47cc93a464a37c2925aca2 +pynvjitlink @ file:///rapids/pynvjitlink-0.3.0-cp312-cp312-linux_x86_64.whl#sha256=066caf1314730643d3b48c9847bd65789defdd6a7c78993176eb3d8bbe3f4919 +pynvml @ file:///rapids/pynvml-12.0.0-py3-none-any.whl#sha256=fdff84b62a27dbe98e08e1a647eb77342bef1aebe0878bcd15e99a83fcbecb9e +pyparsing==3.2.3 +pytest==8.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==15.1 +pytest-shard==0.1.2 +pytest-xdist==3.7.0 +python-dateutil==2.9.0.post0 +python-hostlist==2.2.1 +python-json-logger==3.3.0 +pytorch-triton @ file:///tmp/dist/pytorch_triton-3.3.0%2Bgit96316ce52.nvinternal-cp312-cp312-linux_x86_64.whl#sha256=6d6921772b9dfd0243c055e0d054de42a113e92cabcfdd1d3e80a5ad9c08cf7d +pytz @ file:///rapids/pytz-2023.4-py2.py3-none-any.whl#sha256=f90ef520d95e7c46951105338d918664ebfd6f1d995bd7d153127ce90efafa6a +PyYAML==6.0.2 +pyzmq==26.4.0 +raft-dask @ file:///rapids/raft_dask-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=36cf37ba6ce35454c4729c2cb597a360f22780eefb11d514a16a07137da5eb6c +rapids-dask-dependency @ file:///rapids/rapids_dask_dependency-25.4.0a0-py3-none-any.whl#sha256=484faca626e7760112a7d8ad3426f5bdd720b609006d8dca87c2607076a348bd +rapids-logger @ file:///rapids/rapids_logger-0.1.18-py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=3617b2d86291ac6cb957542895f20fc9737ace29773ca90337e6fb04c73bd120 +referencing==0.36.2 +regex==2024.11.6 +requests==2.32.3 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rich==14.0.0 +rmm @ file:///rapids/rmm-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1824bcab3836296666980a997ddcb9c4bca24dfb873012cfa0a7ecea583ca712 +rpds-py==0.25.1 +safetensors==0.5.3 +scikit-learn==1.6.1 +scipy==1.15.3 +Send2Trash==1.8.3 +setuptools==78.1.1 +shellingham==1.5.4 +six==1.16.0 +smart-open==7.1.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.13.1 +soupsieve==2.7 +soxr==0.5.0.post1 +spacy==3.7.5 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +srsly==2.5.1 +stack-data==0.6.3 +sympy==1.14.0 +tabulate==0.9.0 +tbb==2021.13.1 +tblib @ file:///rapids/tblib-3.1.0-py3-none-any.whl#sha256=670bb4582578134b3d81a84afa1b016128b429f3d48e6cbbaecc9d15675e984e +tensorboard==2.16.2 +tensorboard-data-server==0.7.2 +tensorrt @ file:///workspace/TensorRT-10.11.0.33/python/tensorrt-10.11.0.33-cp312-none-linux_x86_64.whl#sha256=c2f18f72758a3d54a71901495c8fda9e7094a8bcdd2a5b6b58967fbf05733b3a +terminado==0.18.1 +thinc==8.2.5 +threadpoolctl==3.6.0 +thriftpy2 @ file:///rapids/thriftpy2-0.5.2-cp312-cp312-linux_x86_64.whl#sha256=c0c7d939bdc5ccf7739154959938445d5fdf72f1d6e12bd563947a36f67e2147 +tinycss2==1.4.0 +toolz @ file:///rapids/toolz-1.0.0-py3-none-any.whl#sha256=292c8f1c4e7516bf9086f8850935c799a874039c8bcf959d47b600e4c44a6236 +torch @ file:///opt/transfer/torch-2.8.0a0%2B5228986c39.nv25.6-29946474-cp312-cp312-linux_x86_64.whl#sha256=7a3c9ff5288d8102a31296b3d62491f3f47c8820a5c9aaefc7dc0fe20e1e8ad5 +torch_tensorrt @ file:///opt/pytorch/torch_tensorrt/dist/torch_tensorrt-2.8.0a0-cp312-cp312-linux_x86_64.whl#sha256=26f4d7a04e7e6e017a5b917b8b98b83720bb2bb0195f48d25f52d567d286bfe1 +torchao @ file:///opt/pytorch/ao +torchprofile==0.0.4 +torchvision @ file:///opt/transfer/torchvision-0.22.0a0%2B95f10a4e-29946474-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=746a3c674cc61680a86f4a14b374b21af1b5c21f629c42765b40c9494f6b234d +tornado==6.5.1 +tqdm==4.67.1 +traitlets==5.14.3 +transformer_engine @ file:///pvcdata/aiinfra/b-aghosh/wheels/transformer_engine-2.10.0.dev0%2B10f6128d-cp312-cp312-linux_x86_64.whl#sha256=35cf5f4b83b2baad864e12e512bb422e0929409ac05610812fa521d4942d0189 +treelite @ file:///rapids/treelite-4.4.1-py3-none-linux_x86_64.whl#sha256=174741724205540a1a5b37da81483aeff55d89548f42b3461cf07acac7ae96eb +typer==0.16.0 +types-dataclasses==0.6.6 +types-python-dateutil==2.9.0.20250516 +typing-inspection==0.4.1 +typing_extensions==4.14.0 +tzdata @ file:///rapids/tzdata-2025.2-py2.py3-none-any.whl#sha256=1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 +ucx-py @ file:///rapids/ucx_py-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=4a452fd8d8c983283ada3d49086e3657dc2ee94fbad08efdd708d15ab6e0b73a +ucxx @ file:///rapids/ucxx-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=f1b4e55cb45c66fe88a2d8f9c2b87751165f3cfec21fbdc9c47c4ddff75e8a5f +-e git+https://github.com/Dots-Infra/UltraEP.git@94cab099b44fffa99a82fea99e7c12d89cf65e4f#egg=ultra_ep +uri-template==1.3.0 +urllib3==2.0.7 +wasabi==1.1.3 +wcwidth==0.2.13 +weasel==0.4.1 +webcolors==24.11.1 +webencodings==0.5.1 +websocket-client==1.8.0 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==1.17.2 +xdoctest==1.0.2 +xgboost @ file:///rapids/xgboost-2.1.4-py3-none-manylinux_2_39_x86_64.whl#sha256=25eeca27f4048fc491940b3a0b4b5782ee48c33cb3aec059cef71f8da542930b +xxhash==4.0.1 +yarl @ file:///rapids/yarl-1.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=bf099e2432131093cc611623e0b0bcc399b8cddd9a91eded8bfb50402ec35018 +zarr @ file:///rapids/zarr-2.18.7-py3-none-any.whl#sha256=ac3dc4033e9ae4e9d7b5e27c97ea3eaf1003cc0a07f010bd83d5134bf8c4b223 +zict @ file:///rapids/zict-3.0.0-py2.py3-none-any.whl#sha256=5796e36bd0e0cc8cf0fbc1ace6a68912611c1dbd74750a3f3026b9b9d6a327ae +zipp==3.22.0 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/provenance.txt b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/provenance.txt new file mode 100644 index 0000000000000000000000000000000000000000..81e6e8c967a02552541f0bac4c56929e220fd867 --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/provenance.txt @@ -0,0 +1,90 @@ +============================================================== +PROVENANCE: ultraep__tierbprobe__gbs16 +============================================================== +captured_at : 2026-08-30T05:06:44+00:00 +hostname : sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0 +kernel : 6.8.0-1043-nvidia +cwd : /workspace/blackwell_run_scripts +command : run_ladder.sh --phase tierbprobe --gbs 16 --out /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe --nsys + +---- GPU / driver -------------------------------------------- + 0, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 1, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 2, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 3, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 4, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 5, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 6, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + 7, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 + persistence/clocks: + 0, 120 MHz, 1965 MHz, 1000.00 W, 29 + 1, 120 MHz, 1965 MHz, 1000.00 W, 34 + 2, 120 MHz, 1965 MHz, 1000.00 W, 34 + 3, 120 MHz, 1965 MHz, 1000.00 W, 29 + 4, 120 MHz, 1965 MHz, 1000.00 W, 30 + 5, 120 MHz, 1965 MHz, 1000.00 W, 34 + 6, 120 MHz, 1965 MHz, 1000.00 W, 34 + 7, 120 MHz, 1965 MHz, 1000.00 W, 29 + + NVLink topology: + GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID + GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 PIX NODE SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 NODE PIX SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 SYS SYS PIX NODE SYS SYS SYS SYS SYS 28-55 1 N/A + GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 SYS SYS NODE PIX SYS SYS SYS SYS SYS 28-55 1 N/A + GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 SYS SYS SYS SYS PIX NODE SYS SYS SYS 56-83 2 N/A + GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 SYS SYS SYS SYS NODE PIX SYS SYS SYS 56-83 2 N/A + GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 SYS SYS SYS SYS SYS SYS PIX NODE SYS 84-111 3 N/A + GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X SYS SYS SYS SYS SYS SYS NODE PIX SYS 84-111 3 N/A + NIC0 PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS SYS NODE + NIC1 NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS SYS NODE + NIC2 SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS + NIC3 SYS SYS NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS + NIC4 SYS SYS SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS + unavailable + +---- toolchain ----------------------------------------------- + nvcc : release 12.9, V12.9.86 + gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 + python : 3.12.3 + torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3 + torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120'] + transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py] + deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py] + ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py] + moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py] + megatron : 0.14.0rc5 [None] + flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py] + numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py] + huggingface_hub: 1.29.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py] + +---- repositories -------------------------------------------- + Megatron-MoE-Echo token-drop-analysis b099fd8494b0 + Megatron-UltraEP ultraep-token-drop-analysis eaf829e7ad21 + UltraEP main 94cab099b44f + DeepEP HEAD 34152ae28f80 (+3 untracked: build output) + MoonEP megatron-ddp-symmetric-params d8bc6667fad7 + blackwell_run_scripts granularity-ultraep 818ab62ae4af + +---- environment --------------------------------------------- + CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0 + CUDA_DRIVER_VERSION=575.57.08 + CUDA_HOME=/usr/local/cuda + CUDA_MODULE_LOADING=LAZY + CUDA_VERSION=12.9.1.010 + HF_REMOTE_ROOT=tiera-rerun-fixed-moonep + NCCL_VERSION=2.27.3 + NVSHMEM_VERSION=3.2.5 + PYTORCH_BUILD_NUMBER=0 + PYTORCH_BUILD_VERSION=2.8.0a0+5228986 + PYTORCH_HOME=/opt/pytorch/pytorch + PYTORCH_VERSION=2.8.0a0+5228986 + TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1 + TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX + TORCH_NCCL_USE_COMM_NONBLOCKING=0 + +---- shared library resolution ------------------------------- + libnvshmem_host.so: not in ldconfig + libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2 + lrwxrwxrwx 1 root root 20 Aug 29 16:41 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3 + -rw-r--r-- 1 root root 179045744 Aug 29 16:37 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..2b28b7f37df2ade12e43fc61fbea79e2612c8cd7 --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 2 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 819 +1, 819 +2, 819 +3, 819 +4, 819 +5, 819 +6, 819 +7, 819 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 2231 +1, 2231 +2, 2231 +3, 2231 +4, 2231 +5, 2231 +6, 2231 +7, 2231 +0, 2903 +1, 2903 +2, 2903 +3, 2903 +4, 2903 +5, 2903 +6, 2903 +7, 2903 +0, 42619 +1, 42635 +2, 42621 +3, 42607 +4, 42641 +5, 42607 +6, 42639 +7, 42645 +0, 43369 +1, 43357 +2, 43367 +3, 43355 +4, 43365 +5, 43367 +6, 43367 +7, 43349 +0, 44133 +1, 44131 +2, 44131 +3, 44131 +4, 44131 +5, 44131 +6, 44131 +7, 44131 +0, 44133 +1, 44131 +2, 44131 +3, 44131 +4, 44131 +5, 44131 +6, 44131 +7, 44131 +0, 44693 +1, 44691 +2, 44691 +3, 44691 +4, 44691 +5, 44691 +6, 44691 +7, 44691 +0, 44693 +1, 44691 +2, 44691 +3, 44691 +4, 44691 +5, 44691 +6, 44691 +7, 44691 +0, 70099 +1, 70097 +2, 70097 +3, 70097 +4, 70097 +5, 70097 +6, 70097 +7, 70097 +0, 70659 +1, 70657 +2, 70657 +3, 70657 +4, 70657 +5, 70657 +6, 70657 +7, 70657 +0, 70659 +1, 70657 +2, 70657 +3, 70657 +4, 70657 +5, 70657 +6, 70657 +7, 70657 +0, 82519 +1, 82519 +2, 82511 +3, 82523 +4, 82723 +5, 82529 +6, 82541 +7, 82551 +0, 83485 +1, 83483 +2, 83483 +3, 83483 +4, 83651 +5, 83483 +6, 83483 +7, 83483 +0, 89743 +1, 89921 +2, 90033 +3, 89937 +4, 90403 +5, 89971 +6, 89993 +7, 90015 +0, 107979 +1, 107945 +2, 107945 +3, 107945 +4, 107889 +5, 107945 +6, 107945 +7, 107945 +0, 107979 +1, 108731 +2, 108731 +3, 108731 +4, 108675 +5, 108731 +6, 108731 +7, 108731 +0, 107979 +1, 108731 +2, 108731 +3, 108731 +4, 108675 +5, 108731 +6, 108731 +7, 108731 +0, 107979 +1, 108731 +2, 108731 +3, 108731 +4, 108675 +5, 108731 +6, 108731 +7, 108731 +0, 107979 +1, 108731 +2, 108731 +3, 108731 +4, 108675 +5, 108731 +6, 108731 +7, 108731 +0, 107979 +1, 108731 +2, 108731 +3, 108731 +4, 108675 +5, 108731 +6, 108731 +7, 108731 +0, 103595 +1, 104609 +2, 104609 +3, 104609 +4, 104553 +5, 104673 +6, 104933 +7, 104929 +0, 193 +1, 104606 +2, 963 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1/run.env b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..629416f07ce73521b49fd24e3d1ce7ab35c947ac --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1/run.env @@ -0,0 +1,70 @@ +backend=ultraep +config=G1 +family=tierA +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=14336 +num_experts=8 +topk=2 +experts_per_rank=1 +expert_params=176160768 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=8192 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=1 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=1024 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1/train.log b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..049e2a6b264da04b63bc73bafb3d9f37c7dfdf06 --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1/train.log @@ -0,0 +1,8834 @@ +[05:06:59] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + G1 / ultraep (slots/rank=1 policy=matched) + E=8 K=2 H'=14336 m=1 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=static cap=1.25 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0830 05:07:00.747000 478477 torch/distributed/run.py:766] +W0830 05:07:00.747000 478477 torch/distributed/run.py:766] ***************************************** +W0830 05:07:00.747000 478477 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 05:07:00.747000 478477 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 14336 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 1 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 2 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 8 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 134 is connected to 5 is connected to is connected to 7 is connected to 77 peer ranks. 7 peer ranks. peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : 77[Gloo] Rank +7 + +[Gloo] Rank +67 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to [Gloo] Rank 7 peer ranks. 0Expected number of connected peer ranks is : is connected to 77 + peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 47 is connected to +7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 77 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank7]:[W830 05:07:18.699268635 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 05:07:18.699306360 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 05:07:18.699354872 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 05:07:18.699471588 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 05:07:18.700492592 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 05:07:18.702641922 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 05:07:18.702815830 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 05:07:18.878067840 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.836 seconds +time to initialize megatron (seconds): 7.415 +[after megatron is initialized] datetime: 2026-08-30 05:07:24 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 6720983040 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (41988096 elements, 41988096 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_proj.weight +Params for bucket 3 (41984000 elements, 41984000 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.pre_mlp_layernorm.weight +Params for bucket 4 (41984000 elements, 41984000 padded size): + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.pre_mlp_layernorm.weight +Params for bucket 5 (41984000 elements, 41984000 padded size): + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.pre_mlp_layernorm.weight +Params for bucket 6 (41984000 elements, 41984000 padded size): + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.pre_mlp_layernorm.weight +Params for bucket 7 (41984000 elements, 41984000 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.pre_mlp_layernorm.weight +Params for bucket 8 (41984000 elements, 41984000 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_proj.weight +Params for bucket 9 (41984000 elements, 41984000 padded size): + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 10 (41984000 elements, 41984000 padded size): + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight +Params for bucket 11 (41984000 elements, 41984000 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_proj.weight +Params for bucket 12 (41984000 elements, 41984000 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.pre_mlp_layernorm.weight +Params for bucket 13 (41984000 elements, 41984000 padded size): + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 14 (41984000 elements, 41984000 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.pre_mlp_layernorm.weight +Params for bucket 15 (41984000 elements, 41984000 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.pre_mlp_layernorm.weight +Params for bucket 16 (41984000 elements, 41984000 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.pre_mlp_layernorm.weight +Params for bucket 17 (41984000 elements, 41984000 padded size): + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 32 +Params for bucket 1 (58720256 elements, 58720256 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 +Params for bucket 2 (117440512 elements, 117440512 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 +Params for bucket 3 (58720256 elements, 58720256 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 +Params for bucket 4 (117440512 elements, 117440512 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 +Params for bucket 5 (58720256 elements, 58720256 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 +Params for bucket 6 (117440512 elements, 117440512 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 +Params for bucket 7 (58720256 elements, 58720256 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 +Params for bucket 8 (117440512 elements, 117440512 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 +Params for bucket 9 (58720256 elements, 58720256 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 +Params for bucket 10 (117440512 elements, 117440512 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 +Params for bucket 11 (58720256 elements, 58720256 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 +Params for bucket 12 (117440512 elements, 117440512 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 13 (58720256 elements, 58720256 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 +Params for bucket 14 (117440512 elements, 117440512 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 15 (58720256 elements, 58720256 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 +Params for bucket 16 (117440512 elements, 117440512 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 17 (58720256 elements, 58720256 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 18 (117440512 elements, 117440512 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 +Params for bucket 19 (58720256 elements, 58720256 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 +Params for bucket 20 (117440512 elements, 117440512 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 +Params for bucket 21 (58720256 elements, 58720256 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 +Params for bucket 22 (117440512 elements, 117440512 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 +Params for bucket 23 (58720256 elements, 58720256 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 +Params for bucket 24 (117440512 elements, 117440512 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 25 (58720256 elements, 58720256 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 +Params for bucket 26 (117440512 elements, 117440512 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 27 (58720256 elements, 58720256 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 +Params for bucket 28 (117440512 elements, 117440512 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 29 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 +Params for bucket 30 (117440512 elements, 117440512 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +Params for bucket 31 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 +Params for bucket 32 (117440512 elements, 117440512 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 05:07:27 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 192 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 05:07:28 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3478.93, 3481.09) + train/valid/test-data-iterators-setup ..........: (77.75, 89.55) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-30 05:07:28 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=10240 expected_recv=16797 expected_overflow=0 exact_recv_per_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=10240 expected_recv=16797 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=10240 expected_recv=17922 expected_overflow=0 exact_recv_per_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=10240 expected_recv=16796 expected_overflow=0 exact_recv_per_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=10240 expected_recv=16779 expected_overflow=0 exact_recv_per_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=10240 expected_recv=16796 expected_overflow=0 exact_recv_per_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=10240 expected_recv=16279 expected_overflow=0 exact_recv_per_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=10240 expected_recv=16798 expected_overflow=0 exact_recv_per_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=10240 expected_recv=17922 actual_valid_recv=18048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=10240 expected_recv=14029 expected_overflow=0 exact_recv_per_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=18048 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=10240 expected_recv=16796 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=10240 expected_recv=16779 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=10240 expected_recv=16796 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=10240 expected_recv=16279 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=10240 expected_recv=16798 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=10240 expected_recv=14029 actual_valid_recv=14208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=10240 expected_recv=17043 expected_overflow=0 exact_recv_per_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=14208 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=10240 expected_recv=17043 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=17152 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=10240 expected_recv=12727 expected_overflow=0 exact_recv_per_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=10240 expected_recv=18473 expected_overflow=0 exact_recv_per_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=10240 expected_recv=12727 actual_valid_recv=12928 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=10240 expected_recv=18472 expected_overflow=0 exact_recv_per_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=10240 expected_recv=18472 expected_overflow=0 exact_recv_per_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=10240 expected_recv=14765 expected_overflow=0 exact_recv_per_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=10240 expected_recv=18473 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=10240 expected_recv=13506 expected_overflow=0 exact_recv_per_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=12928 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=10240 expected_recv=16424 expected_overflow=0 exact_recv_per_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=10240 expected_recv=18472 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=10240 expected_recv=18472 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=18560 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=10240 expected_recv=14765 actual_valid_recv=14976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=10240 expected_recv=16424 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=10240 expected_recv=13506 actual_valid_recv=13696 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=18560 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=18560 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=14976 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=13696 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=10240 expected_recv=11859 expected_overflow=0 exact_recv_per_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=10240 expected_recv=11859 actual_valid_recv=12288 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=10240 expected_recv=17349 expected_overflow=0 exact_recv_per_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=12288 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=10240 expected_recv=16854 expected_overflow=0 exact_recv_per_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=10240 expected_recv=16822 expected_overflow=0 exact_recv_per_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=10240 expected_recv=17349 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=10240 expected_recv=17922 expected_overflow=0 exact_recv_per_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=10240 expected_recv=16021 expected_overflow=0 exact_recv_per_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=10240 expected_recv=17563 expected_overflow=0 exact_recv_per_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=10240 expected_recv=16854 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=17536 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=10240 expected_recv=16822 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=10240 expected_recv=17922 actual_valid_recv=18048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=10240 expected_recv=16021 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=10240 expected_recv=17563 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=17152 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=18048 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=17664 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=10240 expected_recv=15990 expected_overflow=0 exact_recv_per_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=10240 expected_recv=16450 expected_overflow=0 exact_recv_per_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=10240 expected_recv=16512 expected_overflow=0 exact_recv_per_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=10240 expected_recv=15990 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=10240 expected_recv=15479 expected_overflow=0 exact_recv_per_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=10240 expected_recv=16450 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=10240 expected_recv=16512 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=10240 expected_recv=15479 actual_valid_recv=15616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=15616 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=10240 expected_recv=17920 expected_overflow=0 exact_recv_per_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=10240 expected_recv=17920 actual_valid_recv=18048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=18048 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=10240 expected_recv=16163 expected_overflow=0 exact_recv_per_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=10240 expected_recv=16140 expected_overflow=0 exact_recv_per_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=10240 expected_recv=16163 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=10240 expected_recv=13971 expected_overflow=0 exact_recv_per_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=10240 expected_recv=19638 expected_overflow=564 exact_recv_per_rank=[10804, 7046, 3419, 3477, 9024, 10803, 10160, 10803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=10240 expected_recv=17095 expected_overflow=0 exact_recv_per_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=10240 expected_recv=16140 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=10240 expected_recv=13971 actual_valid_recv=14208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=10240 expected_recv=15871 expected_overflow=0 exact_recv_per_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=10240 expected_recv=19638 actual_valid_recv=19200 observed_loss=564 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=10240 expected_recv=17095 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=10240 expected_recv=16867 expected_overflow=0 exact_recv_per_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=14208 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=19200 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=10240 expected_recv=15871 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=10240 expected_recv=16867 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=10240 expected_recv=15666 expected_overflow=0 exact_recv_per_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=10240 expected_recv=21503 expected_overflow=1023 exact_recv_per_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=10240 expected_recv=15666 actual_valid_recv=15744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=10240 expected_recv=21505 expected_overflow=1025 exact_recv_per_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=10240 expected_recv=20335 expected_overflow=546 exact_recv_per_rank=[10786, 5595, 10785, 6706, 8722, 5275, 6881, 10786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=15744 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=10240 expected_recv=21503 actual_valid_recv=20480 observed_loss=1023 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=10240 expected_recv=11030 expected_overflow=0 exact_recv_per_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=10240 expected_recv=17301 expected_overflow=163 exact_recv_per_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=10240 expected_recv=21505 actual_valid_recv=20480 observed_loss=1025 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=20480 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=10240 expected_recv=20335 actual_valid_recv=19840 observed_loss=546 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=10240 expected_recv=14194 expected_overflow=0 exact_recv_per_rank=[10804, 7046, 3419, 3477, 9024, 10803, 10160, 10803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=10240 expected_recv=11030 actual_valid_recv=11264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=20480 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=10240 expected_recv=17301 actual_valid_recv=17280 observed_loss=163 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=19840 slack_slots=640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=11264 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=10240 expected_recv=14194 actual_valid_recv=14464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=17280 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=14464 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=10240 expected_recv=18993 expected_overflow=0 exact_recv_per_rank=[10804, 7046, 3419, 3477, 9024, 10803, 10160, 10803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=10240 expected_recv=18993 actual_valid_recv=19200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=10240 expected_recv=16360 expected_overflow=0 exact_recv_per_rank=[10804, 7046, 3419, 3477, 9024, 10803, 10160, 10803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=19200 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=10240 expected_recv=19636 expected_overflow=563 exact_recv_per_rank=[10804, 7046, 3419, 3477, 9024, 10803, 10160, 10803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=10240 expected_recv=17772 expected_overflow=44 exact_recv_per_rank=[7488, 9179, 7858, 9181, 6603, 8601, 7446, 9180] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=10240 expected_recv=16360 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=10240 expected_recv=19636 expected_overflow=563 exact_recv_per_rank=[10804, 7046, 3419, 3477, 9024, 10803, 10160, 10803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=10240 expected_recv=19636 actual_valid_recv=19200 observed_loss=563 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=10240 expected_recv=12251 expected_overflow=0 exact_recv_per_rank=[10804, 7046, 3419, 3477, 9024, 10803, 10160, 10803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=10240 expected_recv=17772 actual_valid_recv=17792 observed_loss=44 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=10240 expected_recv=12127 expected_overflow=0 exact_recv_per_rank=[10786, 5595, 10785, 6706, 8722, 5275, 6881, 10786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=10240 expected_recv=19636 actual_valid_recv=19200 observed_loss=563 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=19200 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=10240 expected_recv=12251 actual_valid_recv=12416 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=17792 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=19200 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=10240 expected_recv=12127 actual_valid_recv=12288 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=12416 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=10240 expected_recv=14578 expected_overflow=0 exact_recv_per_rank=[10786, 5595, 10785, 6706, 8722, 5275, 6881, 10786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=12288 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=10240 expected_recv=14578 actual_valid_recv=14848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=14848 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=10240 expected_recv=14217 expected_overflow=0 exact_recv_per_rank=[10786, 5595, 10785, 6706, 8722, 5275, 6881, 10786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=10240 expected_recv=20335 expected_overflow=546 exact_recv_per_rank=[10786, 5595, 10785, 6706, 8722, 5275, 6881, 10786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=10240 expected_recv=14217 actual_valid_recv=14336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=10240 expected_recv=18998 expected_overflow=63 exact_recv_per_rank=[8695, 8696, 8697, 5938, 7538, 8696, 8695, 8581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=10240 expected_recv=20335 actual_valid_recv=19840 observed_loss=546 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=10240 expected_recv=14161 expected_overflow=0 exact_recv_per_rank=[10786, 5595, 10785, 6706, 8722, 5275, 6881, 10786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=14336 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=10240 expected_recv=20334 expected_overflow=545 exact_recv_per_rank=[10786, 5595, 10785, 6706, 8722, 5275, 6881, 10786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=10240 expected_recv=18998 actual_valid_recv=18944 observed_loss=63 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=19840 slack_slots=640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=10240 expected_recv=14161 actual_valid_recv=14336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=10240 expected_recv=20456 expected_overflow=1037 exact_recv_per_rank=[7488, 9179, 7858, 9181, 6603, 8601, 7446, 9180] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=10240 expected_recv=16798 expected_overflow=0 exact_recv_per_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=18944 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=10240 expected_recv=20334 actual_valid_recv=19840 observed_loss=545 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=14336 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=10240 expected_recv=17033 expected_overflow=0 exact_recv_per_rank=[7488, 9179, 7858, 9181, 6603, 8601, 7446, 9180] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=10240 expected_recv=20456 actual_valid_recv=19584 observed_loss=1037 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=19840 slack_slots=640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=10240 expected_recv=17033 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=19584 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=10240 expected_recv=14756 expected_overflow=0 exact_recv_per_rank=[7488, 9179, 7858, 9181, 6603, 8601, 7446, 9180] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=17152 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=10240 expected_recv=15400 expected_overflow=0 exact_recv_per_rank=[7488, 9179, 7858, 9181, 6603, 8601, 7446, 9180] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=10240 expected_recv=14756 actual_valid_recv=14976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=10240 expected_recv=16798 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=10240 expected_recv=15400 actual_valid_recv=15488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=14976 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=10240 expected_recv=13951 expected_overflow=0 exact_recv_per_rank=[7488, 9179, 7858, 9181, 6603, 8601, 7446, 9180] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=10240 expected_recv=18629 expected_overflow=0 exact_recv_per_rank=[8498, 8114, 7801, 7165, 8499, 8497, 8462, 8500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=15488 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=10240 expected_recv=11245 expected_overflow=0 exact_recv_per_rank=[7488, 9179, 7858, 9181, 6603, 8601, 7446, 9180] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=10240 expected_recv=13951 actual_valid_recv=14080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=10240 expected_recv=18629 actual_valid_recv=18816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=10240 expected_recv=19864 expected_overflow=928 exact_recv_per_rank=[8695, 8696, 8697, 5938, 7538, 8696, 8695, 8581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=14080 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=10240 expected_recv=11245 actual_valid_recv=11392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=18816 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=10240 expected_recv=19863 expected_overflow=928 exact_recv_per_rank=[8695, 8696, 8697, 5938, 7538, 8696, 8695, 8581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=10240 expected_recv=19864 actual_valid_recv=18944 observed_loss=928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=11392 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=10240 expected_recv=10600 expected_overflow=0 exact_recv_per_rank=[8695, 8696, 8697, 5938, 7538, 8696, 8695, 8581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=10240 expected_recv=19863 actual_valid_recv=18944 observed_loss=928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=18944 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=10240 expected_recv=15703 expected_overflow=0 exact_recv_per_rank=[8695, 8696, 8697, 5938, 7538, 8696, 8695, 8581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=10240 expected_recv=10600 actual_valid_recv=10752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=18944 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=10240 expected_recv=15703 actual_valid_recv=15872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=10752 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=15872 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=10240 expected_recv=18373 expected_overflow=0 exact_recv_per_rank=[8695, 8696, 8697, 5938, 7538, 8696, 8695, 8581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=10240 expected_recv=16735 expected_overflow=0 exact_recv_per_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=10240 expected_recv=17582 expected_overflow=0 exact_recv_per_rank=[8686, 8413, 7684, 8687, 6971, 8686, 7724, 8685] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=10240 expected_recv=18373 actual_valid_recv=18432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=10240 expected_recv=16883 expected_overflow=0 exact_recv_per_rank=[8695, 8696, 8697, 5938, 7538, 8696, 8695, 8581] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=10240 expected_recv=16735 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=10240 expected_recv=17582 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=18432 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=10240 expected_recv=17270 expected_overflow=0 exact_recv_per_rank=[8498, 8114, 7801, 7165, 8499, 8497, 8462, 8500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=10240 expected_recv=12300 expected_overflow=0 exact_recv_per_rank=[8498, 8114, 7801, 7165, 8499, 8497, 8462, 8500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=10240 expected_recv=16883 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=17664 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=10240 expected_recv=17628 expected_overflow=0 exact_recv_per_rank=[8498, 8114, 7801, 7165, 8499, 8497, 8462, 8500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=10240 expected_recv=17270 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=10240 expected_recv=12300 actual_valid_recv=12416 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=10240 expected_recv=18631 expected_overflow=0 exact_recv_per_rank=[8498, 8114, 7801, 7165, 8499, 8497, 8462, 8500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=17152 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=10240 expected_recv=17628 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=12416 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=10240 expected_recv=18631 actual_valid_recv=18816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=17792 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=18816 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=10240 expected_recv=18626 expected_overflow=0 exact_recv_per_rank=[8498, 8114, 7801, 7165, 8499, 8497, 8462, 8500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=10240 expected_recv=23259 expected_overflow=3888 exact_recv_per_rank=[14128, 5065, 1627, 14128, 5565, 4323, 7460, 13240] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=10240 expected_recv=17561 expected_overflow=0 exact_recv_per_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=10240 expected_recv=18626 actual_valid_recv=18816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=10240 expected_recv=10702 expected_overflow=0 exact_recv_per_rank=[8498, 8114, 7801, 7165, 8499, 8497, 8462, 8500] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=10240 expected_recv=23259 actual_valid_recv=19456 observed_loss=3888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=10240 expected_recv=17308 expected_overflow=0 exact_recv_per_rank=[8686, 8413, 7684, 8687, 6971, 8686, 7724, 8685] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=18816 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=10240 expected_recv=16619 expected_overflow=0 exact_recv_per_rank=[8686, 8413, 7684, 8687, 6971, 8686, 7724, 8685] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=10240 expected_recv=15398 expected_overflow=0 exact_recv_per_rank=[8686, 8413, 7684, 8687, 6971, 8686, 7724, 8685] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=10240 expected_recv=17561 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=10240 expected_recv=10702 actual_valid_recv=10880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=19456 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=10240 expected_recv=17308 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=10240 expected_recv=17579 expected_overflow=0 exact_recv_per_rank=[8686, 8413, 7684, 8687, 6971, 8686, 7724, 8685] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=10240 expected_recv=16619 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=10240 expected_recv=15398 actual_valid_recv=15616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=17664 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=10880 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=17536 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=10240 expected_recv=17579 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=15616 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=17920 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=10240 expected_recv=17581 expected_overflow=0 exact_recv_per_rank=[8686, 8413, 7684, 8687, 6971, 8686, 7724, 8685] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=10240 expected_recv=14388 expected_overflow=0 exact_recv_per_rank=[5944, 7831, 8633, 8633, 8633, 8634, 8635, 8593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=10240 expected_recv=17581 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=10240 expected_recv=11423 expected_overflow=0 exact_recv_per_rank=[8686, 8413, 7684, 8687, 6971, 8686, 7724, 8685] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=10240 expected_recv=16051 expected_overflow=0 exact_recv_per_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=10240 expected_recv=14196 expected_overflow=0 exact_recv_per_rank=[14128, 5065, 1627, 14128, 5565, 4323, 7460, 13240] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=10240 expected_recv=14388 actual_valid_recv=14592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=17664 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=10240 expected_recv=12641 expected_overflow=0 exact_recv_per_rank=[14128, 5065, 1627, 14128, 5565, 4323, 7460, 13240] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=10240 expected_recv=14210 expected_overflow=0 exact_recv_per_rank=[14128, 5065, 1627, 14128, 5565, 4323, 7460, 13240] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=10240 expected_recv=11423 actual_valid_recv=11648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=14592 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=10240 expected_recv=14196 actual_valid_recv=14336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=10240 expected_recv=16051 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=10240 expected_recv=12641 actual_valid_recv=12800 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=10240 expected_recv=14210 actual_valid_recv=14336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=11648 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=10240 expected_recv=20906 expected_overflow=3000 exact_recv_per_rank=[14128, 5065, 1627, 14128, 5565, 4323, 7460, 13240] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=14336 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=12800 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=14336 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=10240 expected_recv=20906 actual_valid_recv=18048 observed_loss=3000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=18048 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=10240 expected_recv=11843 expected_overflow=0 exact_recv_per_rank=[14128, 5065, 1627, 14128, 5565, 4323, 7460, 13240] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=10240 expected_recv=15430 expected_overflow=0 exact_recv_per_rank=[6930, 6998, 10911, 9259, 6508, 10912, 10912, 3106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=10240 expected_recv=11843 actual_valid_recv=12032 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=10240 expected_recv=15430 actual_valid_recv=15616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=12032 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=10240 expected_recv=10758 expected_overflow=0 exact_recv_per_rank=[14128, 5065, 1627, 14128, 5565, 4323, 7460, 13240] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=10240 expected_recv=16303 expected_overflow=0 exact_recv_per_rank=[5944, 7831, 8633, 8633, 8633, 8634, 8635, 8593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=10240 expected_recv=17105 expected_overflow=0 exact_recv_per_rank=[5944, 7831, 8633, 8633, 8633, 8634, 8635, 8593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=15616 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=10240 expected_recv=15138 expected_overflow=0 exact_recv_per_rank=[5944, 7831, 8633, 8633, 8633, 8634, 8635, 8593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=10240 expected_recv=15990 expected_overflow=0 exact_recv_per_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=10240 expected_recv=10758 actual_valid_recv=11008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=10240 expected_recv=16303 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=10240 expected_recv=17105 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=10240 expected_recv=17065 expected_overflow=0 exact_recv_per_rank=[5944, 7831, 8633, 8633, 8633, 8634, 8635, 8593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=10240 expected_recv=15138 actual_valid_recv=15488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=11008 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=10240 expected_recv=15990 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=17280 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=15488 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=10240 expected_recv=17065 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=10240 expected_recv=17107 expected_overflow=0 exact_recv_per_rank=[5944, 7831, 8633, 8633, 8633, 8634, 8635, 8593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=10240 expected_recv=16381 expected_overflow=0 exact_recv_per_rank=[8057, 8480, 8480, 8480, 8281, 6798, 8480, 8480] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=10240 expected_recv=17107 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=10240 expected_recv=16381 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=17280 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=10240 expected_recv=15008 expected_overflow=0 exact_recv_per_rank=[6930, 6998, 10911, 9259, 6508, 10912, 10912, 3106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=10240 expected_recv=16862 expected_overflow=0 exact_recv_per_rank=[5944, 7831, 8633, 8633, 8633, 8634, 8635, 8593] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=10240 expected_recv=15498 expected_overflow=0 exact_recv_per_rank=[6930, 6998, 10911, 9259, 6508, 10912, 10912, 3106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=10240 expected_recv=17481 expected_overflow=672 exact_recv_per_rank=[6930, 6998, 10911, 9259, 6508, 10912, 10912, 3106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=10240 expected_recv=11721 expected_overflow=0 exact_recv_per_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=10240 expected_recv=15008 actual_valid_recv=15232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=10240 expected_recv=16862 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=10240 expected_recv=15498 actual_valid_recv=15616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=10240 expected_recv=11454 expected_overflow=0 exact_recv_per_rank=[6930, 6998, 10911, 9259, 6508, 10912, 10912, 3106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=10240 expected_recv=17481 actual_valid_recv=16896 observed_loss=672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=10240 expected_recv=11721 actual_valid_recv=11776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=15232 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=15616 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=10240 expected_recv=11454 actual_valid_recv=11648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=11776 slack_slots=8704 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=11648 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=10240 expected_recv=19412 expected_overflow=672 exact_recv_per_rank=[6930, 6998, 10911, 9259, 6508, 10912, 10912, 3106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=10240 expected_recv=19412 actual_valid_recv=18816 observed_loss=672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=18816 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=10240 expected_recv=16741 expected_overflow=0 exact_recv_per_rank=[8057, 8480, 8480, 8480, 8281, 6798, 8480, 8480] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=10240 expected_recv=17009 expected_overflow=0 exact_recv_per_rank=[8057, 8480, 8480, 8480, 8281, 6798, 8480, 8480] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=10240 expected_recv=19030 expected_overflow=671 exact_recv_per_rank=[6930, 6998, 10911, 9259, 6508, 10912, 10912, 3106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=10240 expected_recv=17010 expected_overflow=0 exact_recv_per_rank=[8057, 8480, 8480, 8480, 8281, 6798, 8480, 8480] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[991, 24884, 5241, 2569, 13382, 3150, 7068, 8251] avg=8192.0000 max=24884 max_rank=1 min=991 min_rank=0 imbalance_ratio=3.037598 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=10240 expected_recv=10364 expected_overflow=0 exact_recv_per_rank=[10804, 7046, 3419, 3477, 9024, 10803, 10160, 10803] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=10240 expected_recv=16741 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=10240 expected_recv=17009 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=10240 expected_recv=17010 expected_overflow=0 exact_recv_per_rank=[8057, 8480, 8480, 8480, 8281, 6798, 8480, 8480] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=10240 expected_recv=19030 actual_valid_recv=18432 observed_loss=671 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=10240 expected_recv=17010 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=10240 expected_recv=10364 actual_valid_recv=10624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=17152 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=10240 expected_recv=17010 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=18432 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=17152 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=10624 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=17152 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=10240 expected_recv=15327 expected_overflow=0 exact_recv_per_rank=[8057, 8480, 8480, 8480, 8281, 6798, 8480, 8480] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] avg=8192.0000 max=8524 max_rank=2 min=6025 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=16360 max_load_reduction_pct=0.657451 moved_to_replicas=41587 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=10240 expected_recv=15327 actual_valid_recv=15488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=15488 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=10240 expected_recv=15831 expected_overflow=0 exact_recv_per_rank=[8057, 8480, 8480, 8480, 8281, 6798, 8480, 8480] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[991, 24884, 5241, 2569, 13382, 3150, 7068, 8251] avg=8192.0000 max=24884 max_rank=1 min=991 min_rank=0 imbalance_ratio=3.037598 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=10240 expected_recv=14985 expected_overflow=0 exact_recv_per_rank=[10786, 5595, 10785, 6706, 8722, 5275, 6881, 10786] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[991, 24884, 5241, 2569, 13382, 3150, 7068, 8251] avg=8192.0000 max=24884 max_rank=1 min=991 min_rank=0 imbalance_ratio=3.037598 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=10240 expected_recv=15831 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=10240 expected_recv=14985 actual_valid_recv=15104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[991, 24884, 5241, 2569, 13382, 3150, 7068, 8251] avg=8192.0000 max=24884 max_rank=1 min=991 min_rank=0 imbalance_ratio=3.037598 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=15104 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[991, 24884, 5241, 2569, 13382, 3150, 7068, 8251] avg=8192.0000 max=24884 max_rank=1 min=991 min_rank=0 imbalance_ratio=3.037598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[8387, 1691, 4070, 7203, 16186, 4440, 11688, 11871] avg=8192.0000 max=16186 max_rank=4 min=1691 min_rank=1 imbalance_ratio=1.975830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] avg=8192.0000 max=8524 max_rank=2 min=6025 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=16360 max_load_reduction_pct=0.657451 moved_to_replicas=41587 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] avg=8192.0000 max=8524 max_rank=2 min=6025 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=16360 max_load_reduction_pct=0.657451 moved_to_replicas=41587 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] avg=8192.0000 max=8524 max_rank=2 min=6025 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=16360 max_load_reduction_pct=0.657451 moved_to_replicas=41587 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[991, 24884, 5241, 2569, 13382, 3150, 7068, 8251] avg=8192.0000 max=24884 max_rank=1 min=991 min_rank=0 imbalance_ratio=3.037598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] avg=8192.0000 max=8939 max_rank=4 min=7003 min_rank=2 imbalance_ratio=1.091187 max_load_reduction=7247 max_load_reduction_pct=0.447733 moved_to_replicas=22777 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] avg=8192.0000 max=8524 max_rank=2 min=6025 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=16360 max_load_reduction_pct=0.657451 moved_to_replicas=41587 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=10240 expected_recv=20459 expected_overflow=1038 exact_recv_per_rank=[7488, 9179, 7858, 9181, 6603, 8601, 7446, 9180] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=10240 expected_recv=20459 actual_valid_recv=19456 observed_loss=1038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[991, 24884, 5241, 2569, 13382, 3150, 7068, 8251] avg=8192.0000 max=24884 max_rank=1 min=991 min_rank=0 imbalance_ratio=3.037598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[8387, 1691, 4070, 7203, 16186, 4440, 11688, 11871] avg=8192.0000 max=16186 max_rank=4 min=1691 min_rank=1 imbalance_ratio=1.975830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[8387, 1691, 4070, 7203, 16186, 4440, 11688, 11871] avg=8192.0000 max=16186 max_rank=4 min=1691 min_rank=1 imbalance_ratio=1.975830 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=19456 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] avg=8192.0000 max=8524 max_rank=2 min=6025 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=16360 max_load_reduction_pct=0.657451 moved_to_replicas=41587 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[8387, 1691, 4070, 7203, 16186, 4440, 11688, 11871] avg=8192.0000 max=16186 max_rank=4 min=1691 min_rank=1 imbalance_ratio=1.975830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[8615, 1037, 7593, 20099, 902, 15779, 9023, 2488] avg=8192.0000 max=20099 max_rank=3 min=902 min_rank=4 imbalance_ratio=2.453491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[8387, 1691, 4070, 7203, 16186, 4440, 11688, 11871] avg=8192.0000 max=16186 max_rank=4 min=1691 min_rank=1 imbalance_ratio=1.975830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] avg=8192.0000 max=8939 max_rank=4 min=7003 min_rank=2 imbalance_ratio=1.091187 max_load_reduction=7247 max_load_reduction_pct=0.447733 moved_to_replicas=22777 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] avg=8192.0000 max=8524 max_rank=2 min=6025 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=16360 max_load_reduction_pct=0.657451 moved_to_replicas=41587 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] avg=8192.0000 max=8939 max_rank=4 min=7003 min_rank=2 imbalance_ratio=1.091187 max_load_reduction=7247 max_load_reduction_pct=0.447733 moved_to_replicas=22777 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=10240 expected_recv=10788 expected_overflow=0 exact_recv_per_rank=[8695, 8696, 8697, 5938, 7538, 8696, 8695, 8581] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] avg=8192.0000 max=8939 max_rank=4 min=7003 min_rank=2 imbalance_ratio=1.091187 max_load_reduction=7247 max_load_reduction_pct=0.447733 moved_to_replicas=22777 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] avg=8192.0000 max=9135 max_rank=4 min=3770 min_rank=1 imbalance_ratio=1.115112 max_load_reduction=10964 max_load_reduction_pct=0.545500 moved_to_replicas=33906 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[8387, 1691, 4070, 7203, 16186, 4440, 11688, 11871] avg=8192.0000 max=16186 max_rank=4 min=1691 min_rank=1 imbalance_ratio=1.975830 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=10240 expected_recv=10788 actual_valid_recv=10880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] avg=8192.0000 max=8939 max_rank=4 min=7003 min_rank=2 imbalance_ratio=1.091187 max_load_reduction=7247 max_load_reduction_pct=0.447733 moved_to_replicas=22777 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=10880 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[8615, 1037, 7593, 20099, 902, 15779, 9023, 2488] avg=8192.0000 max=20099 max_rank=3 min=902 min_rank=4 imbalance_ratio=2.453491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[8615, 1037, 7593, 20099, 902, 15779, 9023, 2488] avg=8192.0000 max=20099 max_rank=3 min=902 min_rank=4 imbalance_ratio=2.453491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[8387, 1691, 4070, 7203, 16186, 4440, 11688, 11871] avg=8192.0000 max=16186 max_rank=4 min=1691 min_rank=1 imbalance_ratio=1.975830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] avg=8192.0000 max=8939 max_rank=4 min=7003 min_rank=2 imbalance_ratio=1.091187 max_load_reduction=7247 max_load_reduction_pct=0.447733 moved_to_replicas=22777 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[5823, 20298, 4359, 6463, 10601, 2673, 9588, 5731] avg=8192.0000 max=20298 max_rank=1 min=2673 min_rank=5 imbalance_ratio=2.477783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] avg=8192.0000 max=9135 max_rank=4 min=3770 min_rank=1 imbalance_ratio=1.115112 max_load_reduction=10964 max_load_reduction_pct=0.545500 moved_to_replicas=33906 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[8615, 1037, 7593, 20099, 902, 15779, 9023, 2488] avg=8192.0000 max=20099 max_rank=3 min=902 min_rank=4 imbalance_ratio=2.453491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[8615, 1037, 7593, 20099, 902, 15779, 9023, 2488] avg=8192.0000 max=20099 max_rank=3 min=902 min_rank=4 imbalance_ratio=2.453491 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=10240 expected_recv=17286 expected_overflow=0 exact_recv_per_rank=[8498, 8114, 7801, 7165, 8499, 8497, 8462, 8500] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] avg=8192.0000 max=8939 max_rank=4 min=7003 min_rank=2 imbalance_ratio=1.091187 max_load_reduction=7247 max_load_reduction_pct=0.447733 moved_to_replicas=22777 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] avg=8192.0000 max=9135 max_rank=4 min=3770 min_rank=1 imbalance_ratio=1.115112 max_load_reduction=10964 max_load_reduction_pct=0.545500 moved_to_replicas=33906 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=10240 expected_recv=17286 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] avg=8192.0000 max=8275 max_rank=5 min=7777 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=12023 max_load_reduction_pct=0.592324 moved_to_replicas=30131 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[8615, 1037, 7593, 20099, 902, 15779, 9023, 2488] avg=8192.0000 max=20099 max_rank=3 min=902 min_rank=4 imbalance_ratio=2.453491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] avg=8192.0000 max=9135 max_rank=4 min=3770 min_rank=1 imbalance_ratio=1.115112 max_load_reduction=10964 max_load_reduction_pct=0.545500 moved_to_replicas=33906 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] avg=8192.0000 max=9135 max_rank=4 min=3770 min_rank=1 imbalance_ratio=1.115112 max_load_reduction=10964 max_load_reduction_pct=0.545500 moved_to_replicas=33906 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[5823, 20298, 4359, 6463, 10601, 2673, 9588, 5731] avg=8192.0000 max=20298 max_rank=1 min=2673 min_rank=5 imbalance_ratio=2.477783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[8615, 1037, 7593, 20099, 902, 15779, 9023, 2488] avg=8192.0000 max=20099 max_rank=3 min=902 min_rank=4 imbalance_ratio=2.453491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[5823, 20298, 4359, 6463, 10601, 2673, 9588, 5731] avg=8192.0000 max=20298 max_rank=1 min=2673 min_rank=5 imbalance_ratio=2.477783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] avg=8192.0000 max=9135 max_rank=4 min=3770 min_rank=1 imbalance_ratio=1.115112 max_load_reduction=10964 max_load_reduction_pct=0.545500 moved_to_replicas=33906 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[12437, 13179, 4062, 7594, 7742, 9524, 5102, 5896] avg=8192.0000 max=13179 max_rank=1 min=4062 min_rank=2 imbalance_ratio=1.608765 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=10240 expected_recv=17582 expected_overflow=0 exact_recv_per_rank=[8686, 8413, 7684, 8687, 6971, 8686, 7724, 8685] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] avg=8192.0000 max=8275 max_rank=5 min=7777 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=12023 max_load_reduction_pct=0.592324 moved_to_replicas=30131 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[5823, 20298, 4359, 6463, 10601, 2673, 9588, 5731] avg=8192.0000 max=20298 max_rank=1 min=2673 min_rank=5 imbalance_ratio=2.477783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[5823, 20298, 4359, 6463, 10601, 2673, 9588, 5731] avg=8192.0000 max=20298 max_rank=1 min=2673 min_rank=5 imbalance_ratio=2.477783 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=10240 expected_recv=17582 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] avg=8192.0000 max=8275 max_rank=5 min=7777 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=12023 max_load_reduction_pct=0.592324 moved_to_replicas=30131 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] avg=8192.0000 max=9135 max_rank=4 min=3770 min_rank=1 imbalance_ratio=1.115112 max_load_reduction=10964 max_load_reduction_pct=0.545500 moved_to_replicas=33906 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=17664 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] avg=8192.0000 max=9525 max_rank=0 min=5896 min_rank=7 imbalance_ratio=1.162720 max_load_reduction=3654 max_load_reduction_pct=0.277259 moved_to_replicas=20495 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] avg=8192.0000 max=8275 max_rank=5 min=7777 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=12023 max_load_reduction_pct=0.592324 moved_to_replicas=30131 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[5823, 20298, 4359, 6463, 10601, 2673, 9588, 5731] avg=8192.0000 max=20298 max_rank=1 min=2673 min_rank=5 imbalance_ratio=2.477783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] avg=8192.0000 max=8275 max_rank=5 min=7777 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=12023 max_load_reduction_pct=0.592324 moved_to_replicas=30131 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[12437, 13179, 4062, 7594, 7742, 9524, 5102, 5896] avg=8192.0000 max=13179 max_rank=1 min=4062 min_rank=2 imbalance_ratio=1.608765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[12437, 13179, 4062, 7594, 7742, 9524, 5102, 5896] avg=8192.0000 max=13179 max_rank=1 min=4062 min_rank=2 imbalance_ratio=1.608765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] avg=8192.0000 max=8275 max_rank=5 min=7777 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=12023 max_load_reduction_pct=0.592324 moved_to_replicas=30131 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=10240 expected_recv=23259 expected_overflow=3888 exact_recv_per_rank=[14128, 5065, 1627, 14128, 5565, 4323, 7460, 13240] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[5823, 20298, 4359, 6463, 10601, 2673, 9588, 5731] avg=8192.0000 max=20298 max_rank=1 min=2673 min_rank=5 imbalance_ratio=2.477783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 1685, 3449, 19120, 2980, 9656, 12376] avg=8192.0000 max=19120 max_rank=4 min=1685 min_rank=2 imbalance_ratio=2.333984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] avg=8192.0000 max=9525 max_rank=0 min=5896 min_rank=7 imbalance_ratio=1.162720 max_load_reduction=3654 max_load_reduction_pct=0.277259 moved_to_replicas=20495 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=10240 expected_recv=23259 actual_valid_recv=19456 observed_loss=3888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[12437, 13179, 4062, 7594, 7742, 9524, 5102, 5896] avg=8192.0000 max=13179 max_rank=1 min=4062 min_rank=2 imbalance_ratio=1.608765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[12437, 13179, 4062, 7594, 7742, 9524, 5102, 5896] avg=8192.0000 max=13179 max_rank=1 min=4062 min_rank=2 imbalance_ratio=1.608765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] avg=8192.0000 max=9525 max_rank=0 min=5896 min_rank=7 imbalance_ratio=1.162720 max_load_reduction=3654 max_load_reduction_pct=0.277259 moved_to_replicas=20495 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=19456 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] avg=8192.0000 max=8275 max_rank=5 min=7777 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=12023 max_load_reduction_pct=0.592324 moved_to_replicas=30131 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] avg=8192.0000 max=10404 max_rank=7 min=3449 min_rank=3 imbalance_ratio=1.270020 max_load_reduction=8716 max_load_reduction_pct=0.455858 moved_to_replicas=13521 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[12437, 13179, 4062, 7594, 7742, 9524, 5102, 5896] avg=8192.0000 max=13179 max_rank=1 min=4062 min_rank=2 imbalance_ratio=1.608765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] avg=8192.0000 max=9525 max_rank=0 min=5896 min_rank=7 imbalance_ratio=1.162720 max_load_reduction=3654 max_load_reduction_pct=0.277259 moved_to_replicas=20495 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 1685, 3449, 19120, 2980, 9656, 12376] avg=8192.0000 max=19120 max_rank=4 min=1685 min_rank=2 imbalance_ratio=2.333984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] avg=8192.0000 max=9525 max_rank=0 min=5896 min_rank=7 imbalance_ratio=1.162720 max_load_reduction=3654 max_load_reduction_pct=0.277259 moved_to_replicas=20495 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 1685, 3449, 19120, 2980, 9656, 12376] avg=8192.0000 max=19120 max_rank=4 min=1685 min_rank=2 imbalance_ratio=2.333984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=10240 expected_recv=17104 expected_overflow=0 exact_recv_per_rank=[5944, 7831, 8633, 8633, 8633, 8634, 8635, 8593] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] avg=8192.0000 max=9525 max_rank=0 min=5896 min_rank=7 imbalance_ratio=1.162720 max_load_reduction=3654 max_load_reduction_pct=0.277259 moved_to_replicas=20495 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[12437, 13179, 4062, 7594, 7742, 9524, 5102, 5896] avg=8192.0000 max=13179 max_rank=1 min=4062 min_rank=2 imbalance_ratio=1.608765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[16295, 1555, 3419, 3477, 1223, 10803, 10160, 18604] avg=8192.0000 max=18604 max_rank=7 min=1223 min_rank=4 imbalance_ratio=2.270996 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=10240 expected_recv=17104 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] avg=8192.0000 max=10404 max_rank=7 min=3449 min_rank=3 imbalance_ratio=1.270020 max_load_reduction=8716 max_load_reduction_pct=0.455858 moved_to_replicas=13521 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=17280 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 1685, 3449, 19120, 2980, 9656, 12376] avg=8192.0000 max=19120 max_rank=4 min=1685 min_rank=2 imbalance_ratio=2.333984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 1685, 3449, 19120, 2980, 9656, 12376] avg=8192.0000 max=19120 max_rank=4 min=1685 min_rank=2 imbalance_ratio=2.333984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] avg=8192.0000 max=10404 max_rank=7 min=3449 min_rank=3 imbalance_ratio=1.270020 max_load_reduction=8716 max_load_reduction_pct=0.455858 moved_to_replicas=13521 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] avg=8192.0000 max=9525 max_rank=0 min=5896 min_rank=7 imbalance_ratio=1.162720 max_load_reduction=3654 max_load_reduction_pct=0.277259 moved_to_replicas=20495 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[10804, 7046, 3419, 3477, 9024, 10803, 10160, 10803] avg=8192.0000 max=10804 max_rank=0 min=3419 min_rank=2 imbalance_ratio=1.318848 max_load_reduction=7800 max_load_reduction_pct=0.419265 moved_to_replicas=32826 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] avg=8192.0000 max=10404 max_rank=7 min=3449 min_rank=3 imbalance_ratio=1.270020 max_load_reduction=8716 max_load_reduction_pct=0.455858 moved_to_replicas=13521 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 1685, 3449, 19120, 2980, 9656, 12376] avg=8192.0000 max=19120 max_rank=4 min=1685 min_rank=2 imbalance_ratio=2.333984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] avg=8192.0000 max=10404 max_rank=7 min=3449 min_rank=3 imbalance_ratio=1.270020 max_load_reduction=8716 max_load_reduction_pct=0.455858 moved_to_replicas=13521 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[16295, 1555, 3419, 3477, 1223, 10803, 10160, 18604] avg=8192.0000 max=18604 max_rank=7 min=1223 min_rank=4 imbalance_ratio=2.270996 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=10240 expected_recv=17759 expected_overflow=0 exact_recv_per_rank=[6930, 6998, 10911, 9259, 6508, 10912, 10912, 3106] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[16295, 1555, 3419, 3477, 1223, 10803, 10160, 18604] avg=8192.0000 max=18604 max_rank=7 min=1223 min_rank=4 imbalance_ratio=2.270996 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=10240 expected_recv=17759 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] avg=8192.0000 max=10404 max_rank=7 min=3449 min_rank=3 imbalance_ratio=1.270020 max_load_reduction=8716 max_load_reduction_pct=0.455858 moved_to_replicas=13521 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=17920 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[14357, 5595, 16727, 6706, 8722, 1704, 939, 10786] avg=8192.0000 max=16727 max_rank=2 min=939 min_rank=6 imbalance_ratio=2.041870 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 1685, 3449, 19120, 2980, 9656, 12376] avg=8192.0000 max=19120 max_rank=4 min=1685 min_rank=2 imbalance_ratio=2.333984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[10804, 7046, 3419, 3477, 9024, 10803, 10160, 10803] avg=8192.0000 max=10804 max_rank=0 min=3419 min_rank=2 imbalance_ratio=1.318848 max_load_reduction=7800 max_load_reduction_pct=0.419265 moved_to_replicas=32826 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[16295, 1555, 3419, 3477, 1223, 10803, 10160, 18604] avg=8192.0000 max=18604 max_rank=7 min=1223 min_rank=4 imbalance_ratio=2.270996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[16295, 1555, 3419, 3477, 1223, 10803, 10160, 18604] avg=8192.0000 max=18604 max_rank=7 min=1223 min_rank=4 imbalance_ratio=2.270996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[10804, 7046, 3419, 3477, 9024, 10803, 10160, 10803] avg=8192.0000 max=10804 max_rank=0 min=3419 min_rank=2 imbalance_ratio=1.318848 max_load_reduction=7800 max_load_reduction_pct=0.419265 moved_to_replicas=32826 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[10786, 5595, 10785, 6706, 8722, 5275, 6881, 10786] avg=8192.0000 max=10786 max_rank=0 min=5275 min_rank=5 imbalance_ratio=1.316650 max_load_reduction=5941 max_load_reduction_pct=0.355174 moved_to_replicas=21432 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] avg=8192.0000 max=10404 max_rank=7 min=3449 min_rank=3 imbalance_ratio=1.270020 max_load_reduction=8716 max_load_reduction_pct=0.455858 moved_to_replicas=13521 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[10804, 7046, 3419, 3477, 9024, 10803, 10160, 10803] avg=8192.0000 max=10804 max_rank=0 min=3419 min_rank=2 imbalance_ratio=1.318848 max_load_reduction=7800 max_load_reduction_pct=0.419265 moved_to_replicas=32826 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[16295, 1555, 3419, 3477, 1223, 10803, 10160, 18604] avg=8192.0000 max=18604 max_rank=7 min=1223 min_rank=4 imbalance_ratio=2.270996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[10804, 7046, 3419, 3477, 9024, 10803, 10160, 10803] avg=8192.0000 max=10804 max_rank=0 min=3419 min_rank=2 imbalance_ratio=1.318848 max_load_reduction=7800 max_load_reduction_pct=0.419265 moved_to_replicas=32826 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=10240 expected_recv=15763 expected_overflow=0 exact_recv_per_rank=[8057, 8480, 8480, 8480, 8281, 6798, 8480, 8480] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[14357, 5595, 16727, 6706, 8722, 1704, 939, 10786] avg=8192.0000 max=16727 max_rank=2 min=939 min_rank=6 imbalance_ratio=2.041870 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=10240 expected_recv=15763 actual_valid_recv=16000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[14357, 5595, 16727, 6706, 8722, 1704, 939, 10786] avg=8192.0000 max=16727 max_rank=2 min=939 min_rank=6 imbalance_ratio=2.041870 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16000 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[10804, 7046, 3419, 3477, 9024, 10803, 10160, 10803] avg=8192.0000 max=10804 max_rank=0 min=3419 min_rank=2 imbalance_ratio=1.318848 max_load_reduction=7800 max_load_reduction_pct=0.419265 moved_to_replicas=32826 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[7488, 3101, 7858, 15863, 6603, 1919, 7446, 15258] avg=8192.0000 max=15863 max_rank=3 min=1919 min_rank=5 imbalance_ratio=1.936401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[10786, 5595, 10785, 6706, 8722, 5275, 6881, 10786] avg=8192.0000 max=10786 max_rank=0 min=5275 min_rank=5 imbalance_ratio=1.316650 max_load_reduction=5941 max_load_reduction_pct=0.355174 moved_to_replicas=21432 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[16295, 1555, 3419, 3477, 1223, 10803, 10160, 18604] avg=8192.0000 max=18604 max_rank=7 min=1223 min_rank=4 imbalance_ratio=2.270996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[14357, 5595, 16727, 6706, 8722, 1704, 939, 10786] avg=8192.0000 max=16727 max_rank=2 min=939 min_rank=6 imbalance_ratio=2.041870 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[10786, 5595, 10785, 6706, 8722, 5275, 6881, 10786] avg=8192.0000 max=10786 max_rank=0 min=5275 min_rank=5 imbalance_ratio=1.316650 max_load_reduction=5941 max_load_reduction_pct=0.355174 moved_to_replicas=21432 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[14357, 5595, 16727, 6706, 8722, 1704, 939, 10786] avg=8192.0000 max=16727 max_rank=2 min=939 min_rank=6 imbalance_ratio=2.041870 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[7488, 9179, 7858, 9181, 6603, 8601, 7446, 9180] avg=8192.0000 max=9181 max_rank=3 min=6603 min_rank=4 imbalance_ratio=1.120728 max_load_reduction=6682 max_load_reduction_pct=0.421232 moved_to_replicas=18181 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[10804, 7046, 3419, 3477, 9024, 10803, 10160, 10803] avg=8192.0000 max=10804 max_rank=0 min=3419 min_rank=2 imbalance_ratio=1.318848 max_load_reduction=7800 max_load_reduction_pct=0.419265 moved_to_replicas=32826 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[10786, 5595, 10785, 6706, 8722, 5275, 6881, 10786] avg=8192.0000 max=10786 max_rank=0 min=5275 min_rank=5 imbalance_ratio=1.316650 max_load_reduction=5941 max_load_reduction_pct=0.355174 moved_to_replicas=21432 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[14357, 5595, 16727, 6706, 8722, 1704, 939, 10786] avg=8192.0000 max=16727 max_rank=2 min=939 min_rank=6 imbalance_ratio=2.041870 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[10786, 5595, 10785, 6706, 8722, 5275, 6881, 10786] avg=8192.0000 max=10786 max_rank=0 min=5275 min_rank=5 imbalance_ratio=1.316650 max_load_reduction=5941 max_load_reduction_pct=0.355174 moved_to_replicas=21432 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[7488, 3101, 7858, 15863, 6603, 1919, 7446, 15258] avg=8192.0000 max=15863 max_rank=3 min=1919 min_rank=5 imbalance_ratio=1.936401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[7488, 3101, 7858, 15863, 6603, 1919, 7446, 15258] avg=8192.0000 max=15863 max_rank=3 min=1919 min_rank=5 imbalance_ratio=1.936401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[991, 24884, 5241, 2569, 13382, 3150, 7068, 8251] avg=8192.0000 max=24884 max_rank=1 min=991 min_rank=0 imbalance_ratio=3.037598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[10786, 5595, 10785, 6706, 8722, 5275, 6881, 10786] avg=8192.0000 max=10786 max_rank=0 min=5275 min_rank=5 imbalance_ratio=1.316650 max_load_reduction=5941 max_load_reduction_pct=0.355174 moved_to_replicas=21432 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[16315, 21035, 925, 1371, 5032, 3582, 9930, 7346] avg=8192.0000 max=21035 max_rank=1 min=925 min_rank=2 imbalance_ratio=2.567749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[7488, 9179, 7858, 9181, 6603, 8601, 7446, 9180] avg=8192.0000 max=9181 max_rank=3 min=6603 min_rank=4 imbalance_ratio=1.120728 max_load_reduction=6682 max_load_reduction_pct=0.421232 moved_to_replicas=18181 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[7488, 3101, 7858, 15863, 6603, 1919, 7446, 15258] avg=8192.0000 max=15863 max_rank=3 min=1919 min_rank=5 imbalance_ratio=1.936401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[14357, 5595, 16727, 6706, 8722, 1704, 939, 10786] avg=8192.0000 max=16727 max_rank=2 min=939 min_rank=6 imbalance_ratio=2.041870 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[7488, 9179, 7858, 9181, 6603, 8601, 7446, 9180] avg=8192.0000 max=9181 max_rank=3 min=6603 min_rank=4 imbalance_ratio=1.120728 max_load_reduction=6682 max_load_reduction_pct=0.421232 moved_to_replicas=18181 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[7488, 3101, 7858, 15863, 6603, 1919, 7446, 15258] avg=8192.0000 max=15863 max_rank=3 min=1919 min_rank=5 imbalance_ratio=1.936401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] avg=8192.0000 max=8524 max_rank=2 min=6025 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=16360 max_load_reduction_pct=0.657451 moved_to_replicas=41587 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[8695, 8696, 8697, 5938, 7538, 8696, 8695, 8581] avg=8192.0000 max=8697 max_rank=2 min=5938 min_rank=3 imbalance_ratio=1.061646 max_load_reduction=12338 max_load_reduction_pct=0.586546 moved_to_replicas=28653 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[7488, 9179, 7858, 9181, 6603, 8601, 7446, 9180] avg=8192.0000 max=9181 max_rank=3 min=6603 min_rank=4 imbalance_ratio=1.120728 max_load_reduction=6682 max_load_reduction_pct=0.421232 moved_to_replicas=18181 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[16315, 21035, 925, 1371, 5032, 3582, 9930, 7346] avg=8192.0000 max=21035 max_rank=1 min=925 min_rank=2 imbalance_ratio=2.567749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[10786, 5595, 10785, 6706, 8722, 5275, 6881, 10786] avg=8192.0000 max=10786 max_rank=0 min=5275 min_rank=5 imbalance_ratio=1.316650 max_load_reduction=5941 max_load_reduction_pct=0.355174 moved_to_replicas=21432 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[7488, 3101, 7858, 15863, 6603, 1919, 7446, 15258] avg=8192.0000 max=15863 max_rank=3 min=1919 min_rank=5 imbalance_ratio=1.936401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[7488, 9179, 7858, 9181, 6603, 8601, 7446, 9180] avg=8192.0000 max=9181 max_rank=3 min=6603 min_rank=4 imbalance_ratio=1.120728 max_load_reduction=6682 max_load_reduction_pct=0.421232 moved_to_replicas=18181 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[16315, 21035, 925, 1371, 5032, 3582, 9930, 7346] avg=8192.0000 max=21035 max_rank=1 min=925 min_rank=2 imbalance_ratio=2.567749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[8695, 8696, 8697, 5938, 7538, 8696, 8695, 8581] avg=8192.0000 max=8697 max_rank=2 min=5938 min_rank=3 imbalance_ratio=1.061646 max_load_reduction=12338 max_load_reduction_pct=0.586546 moved_to_replicas=28653 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[8387, 1691, 4070, 7203, 16186, 4440, 11688, 11871] avg=8192.0000 max=16186 max_rank=4 min=1691 min_rank=1 imbalance_ratio=1.975830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[22351, 8114, 4267, 7165, 1125, 2738, 1983, 17793] avg=8192.0000 max=22351 max_rank=0 min=1125 min_rank=4 imbalance_ratio=2.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[16315, 21035, 925, 1371, 5032, 3582, 9930, 7346] avg=8192.0000 max=21035 max_rank=1 min=925 min_rank=2 imbalance_ratio=2.567749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[8695, 8696, 8697, 5938, 7538, 8696, 8695, 8581] avg=8192.0000 max=8697 max_rank=2 min=5938 min_rank=3 imbalance_ratio=1.061646 max_load_reduction=12338 max_load_reduction_pct=0.586546 moved_to_replicas=28653 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[7488, 9179, 7858, 9181, 6603, 8601, 7446, 9180] avg=8192.0000 max=9181 max_rank=3 min=6603 min_rank=4 imbalance_ratio=1.120728 max_load_reduction=6682 max_load_reduction_pct=0.421232 moved_to_replicas=18181 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[16315, 21035, 925, 1371, 5032, 3582, 9930, 7346] avg=8192.0000 max=21035 max_rank=1 min=925 min_rank=2 imbalance_ratio=2.567749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[8498, 8114, 7801, 7165, 8499, 8497, 8462, 8500] avg=8192.0000 max=8500 max_rank=7 min=7165 min_rank=3 imbalance_ratio=1.037598 max_load_reduction=13851 max_load_reduction_pct=0.619704 moved_to_replicas=38887 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[7488, 3101, 7858, 15863, 6603, 1919, 7446, 15258] avg=8192.0000 max=15863 max_rank=3 min=1919 min_rank=5 imbalance_ratio=1.936401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[8695, 8696, 8697, 5938, 7538, 8696, 8695, 8581] avg=8192.0000 max=8697 max_rank=2 min=5938 min_rank=3 imbalance_ratio=1.061646 max_load_reduction=12338 max_load_reduction_pct=0.586546 moved_to_replicas=28653 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] avg=8192.0000 max=8939 max_rank=4 min=7003 min_rank=2 imbalance_ratio=1.091187 max_load_reduction=7247 max_load_reduction_pct=0.447733 moved_to_replicas=22777 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[22351, 8114, 4267, 7165, 1125, 2738, 1983, 17793] avg=8192.0000 max=22351 max_rank=0 min=1125 min_rank=4 imbalance_ratio=2.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[8695, 8696, 8697, 5938, 7538, 8696, 8695, 8581] avg=8192.0000 max=8697 max_rank=2 min=5938 min_rank=3 imbalance_ratio=1.061646 max_load_reduction=12338 max_load_reduction_pct=0.586546 moved_to_replicas=28653 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[22351, 8114, 4267, 7165, 1125, 2738, 1983, 17793] avg=8192.0000 max=22351 max_rank=0 min=1125 min_rank=4 imbalance_ratio=2.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[16315, 21035, 925, 1371, 5032, 3582, 9930, 7346] avg=8192.0000 max=21035 max_rank=1 min=925 min_rank=2 imbalance_ratio=2.567749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[7488, 9179, 7858, 9181, 6603, 8601, 7446, 9180] avg=8192.0000 max=9181 max_rank=3 min=6603 min_rank=4 imbalance_ratio=1.120728 max_load_reduction=6682 max_load_reduction_pct=0.421232 moved_to_replicas=18181 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[8498, 8114, 7801, 7165, 8499, 8497, 8462, 8500] avg=8192.0000 max=8500 max_rank=7 min=7165 min_rank=3 imbalance_ratio=1.037598 max_load_reduction=13851 max_load_reduction_pct=0.619704 moved_to_replicas=38887 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[22351, 8114, 4267, 7165, 1125, 2738, 1983, 17793] avg=8192.0000 max=22351 max_rank=0 min=1125 min_rank=4 imbalance_ratio=2.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[9580, 5688, 3486, 20515, 6077, 11411, 7724, 1055] avg=8192.0000 max=20515 max_rank=3 min=1055 min_rank=7 imbalance_ratio=2.504272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[8615, 1037, 7593, 20099, 902, 15779, 9023, 2488] avg=8192.0000 max=20099 max_rank=3 min=902 min_rank=4 imbalance_ratio=2.453491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[8498, 8114, 7801, 7165, 8499, 8497, 8462, 8500] avg=8192.0000 max=8500 max_rank=7 min=7165 min_rank=3 imbalance_ratio=1.037598 max_load_reduction=13851 max_load_reduction_pct=0.619704 moved_to_replicas=38887 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[8695, 8696, 8697, 5938, 7538, 8696, 8695, 8581] avg=8192.0000 max=8697 max_rank=2 min=5938 min_rank=3 imbalance_ratio=1.061646 max_load_reduction=12338 max_load_reduction_pct=0.586546 moved_to_replicas=28653 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[22351, 8114, 4267, 7165, 1125, 2738, 1983, 17793] avg=8192.0000 max=22351 max_rank=0 min=1125 min_rank=4 imbalance_ratio=2.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[8498, 8114, 7801, 7165, 8499, 8497, 8462, 8500] avg=8192.0000 max=8500 max_rank=7 min=7165 min_rank=3 imbalance_ratio=1.037598 max_load_reduction=13851 max_load_reduction_pct=0.619704 moved_to_replicas=38887 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[8686, 8413, 7684, 8687, 6971, 8686, 7724, 8685] avg=8192.0000 max=8687 max_rank=3 min=6971 min_rank=4 imbalance_ratio=1.060425 max_load_reduction=11828 max_load_reduction_pct=0.576554 moved_to_replicas=33656 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[16315, 21035, 925, 1371, 5032, 3582, 9930, 7346] avg=8192.0000 max=21035 max_rank=1 min=925 min_rank=2 imbalance_ratio=2.567749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] avg=8192.0000 max=9135 max_rank=4 min=3770 min_rank=1 imbalance_ratio=1.115112 max_load_reduction=10964 max_load_reduction_pct=0.545500 moved_to_replicas=33906 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[9580, 5688, 3486, 20515, 6077, 11411, 7724, 1055] avg=8192.0000 max=20515 max_rank=3 min=1055 min_rank=7 imbalance_ratio=2.504272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[8498, 8114, 7801, 7165, 8499, 8497, 8462, 8500] avg=8192.0000 max=8500 max_rank=7 min=7165 min_rank=3 imbalance_ratio=1.037598 max_load_reduction=13851 max_load_reduction_pct=0.619704 moved_to_replicas=38887 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[9580, 5688, 3486, 20515, 6077, 11411, 7724, 1055] avg=8192.0000 max=20515 max_rank=3 min=1055 min_rank=7 imbalance_ratio=2.504272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[22351, 8114, 4267, 7165, 1125, 2738, 1983, 17793] avg=8192.0000 max=22351 max_rank=0 min=1125 min_rank=4 imbalance_ratio=2.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[8695, 8696, 8697, 5938, 7538, 8696, 8695, 8581] avg=8192.0000 max=8697 max_rank=2 min=5938 min_rank=3 imbalance_ratio=1.061646 max_load_reduction=12338 max_load_reduction_pct=0.586546 moved_to_replicas=28653 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[8686, 8413, 7684, 8687, 6971, 8686, 7724, 8685] avg=8192.0000 max=8687 max_rank=3 min=6971 min_rank=4 imbalance_ratio=1.060425 max_load_reduction=11828 max_load_reduction_pct=0.576554 moved_to_replicas=33656 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[9580, 5688, 3486, 20515, 6077, 11411, 7724, 1055] avg=8192.0000 max=20515 max_rank=3 min=1055 min_rank=7 imbalance_ratio=2.504272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[14128, 5065, 1627, 20442, 5565, 4323, 1146, 13240] avg=8192.0000 max=20442 max_rank=3 min=1146 min_rank=6 imbalance_ratio=2.495361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[8686, 8413, 7684, 8687, 6971, 8686, 7724, 8685] avg=8192.0000 max=8687 max_rank=3 min=6971 min_rank=4 imbalance_ratio=1.060425 max_load_reduction=11828 max_load_reduction_pct=0.576554 moved_to_replicas=33656 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[8498, 8114, 7801, 7165, 8499, 8497, 8462, 8500] avg=8192.0000 max=8500 max_rank=7 min=7165 min_rank=3 imbalance_ratio=1.037598 max_load_reduction=13851 max_load_reduction_pct=0.619704 moved_to_replicas=38887 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[5823, 20298, 4359, 6463, 10601, 2673, 9588, 5731] avg=8192.0000 max=20298 max_rank=1 min=2673 min_rank=5 imbalance_ratio=2.477783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[9580, 5688, 3486, 20515, 6077, 11411, 7724, 1055] avg=8192.0000 max=20515 max_rank=3 min=1055 min_rank=7 imbalance_ratio=2.504272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[8686, 8413, 7684, 8687, 6971, 8686, 7724, 8685] avg=8192.0000 max=8687 max_rank=3 min=6971 min_rank=4 imbalance_ratio=1.060425 max_load_reduction=11828 max_load_reduction_pct=0.576554 moved_to_replicas=33656 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[14128, 5065, 1627, 14128, 5565, 4323, 7460, 13240] avg=8192.0000 max=14128 max_rank=0 min=1627 min_rank=2 imbalance_ratio=1.724609 max_load_reduction=6314 max_load_reduction_pct=0.308874 moved_to_replicas=17674 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[22351, 8114, 4267, 7165, 1125, 2738, 1983, 17793] avg=8192.0000 max=22351 max_rank=0 min=1125 min_rank=4 imbalance_ratio=2.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[14128, 5065, 1627, 20442, 5565, 4323, 1146, 13240] avg=8192.0000 max=20442 max_rank=3 min=1146 min_rank=6 imbalance_ratio=2.495361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] avg=8192.0000 max=8275 max_rank=5 min=7777 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=12023 max_load_reduction_pct=0.592324 moved_to_replicas=30131 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[8686, 8413, 7684, 8687, 6971, 8686, 7724, 8685] avg=8192.0000 max=8687 max_rank=3 min=6971 min_rank=4 imbalance_ratio=1.060425 max_load_reduction=11828 max_load_reduction_pct=0.576554 moved_to_replicas=33656 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[14128, 5065, 1627, 20442, 5565, 4323, 1146, 13240] avg=8192.0000 max=20442 max_rank=3 min=1146 min_rank=6 imbalance_ratio=2.495361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[9580, 5688, 3486, 20515, 6077, 11411, 7724, 1055] avg=8192.0000 max=20515 max_rank=3 min=1055 min_rank=7 imbalance_ratio=2.504272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[14128, 5065, 1627, 14128, 5565, 4323, 7460, 13240] avg=8192.0000 max=14128 max_rank=0 min=1627 min_rank=2 imbalance_ratio=1.724609 max_load_reduction=6314 max_load_reduction_pct=0.308874 moved_to_replicas=17674 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[8498, 8114, 7801, 7165, 8499, 8497, 8462, 8500] avg=8192.0000 max=8500 max_rank=7 min=7165 min_rank=3 imbalance_ratio=1.037598 max_load_reduction=13851 max_load_reduction_pct=0.619704 moved_to_replicas=38887 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[14128, 5065, 1627, 20442, 5565, 4323, 1146, 13240] avg=8192.0000 max=20442 max_rank=3 min=1146 min_rank=6 imbalance_ratio=2.495361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[14128, 5065, 1627, 14128, 5565, 4323, 7460, 13240] avg=8192.0000 max=14128 max_rank=0 min=1627 min_rank=2 imbalance_ratio=1.724609 max_load_reduction=6314 max_load_reduction_pct=0.308874 moved_to_replicas=17674 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[3811, 4843, 11621, 17947, 10527, 7640, 1454, 7693] avg=8192.0000 max=17947 max_rank=3 min=1454 min_rank=6 imbalance_ratio=2.190796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[8686, 8413, 7684, 8687, 6971, 8686, 7724, 8685] avg=8192.0000 max=8687 max_rank=3 min=6971 min_rank=4 imbalance_ratio=1.060425 max_load_reduction=11828 max_load_reduction_pct=0.576554 moved_to_replicas=33656 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[12437, 13179, 4062, 7594, 7742, 9524, 5102, 5896] avg=8192.0000 max=13179 max_rank=1 min=4062 min_rank=2 imbalance_ratio=1.608765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[14128, 5065, 1627, 20442, 5565, 4323, 1146, 13240] avg=8192.0000 max=20442 max_rank=3 min=1146 min_rank=6 imbalance_ratio=2.495361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[14128, 5065, 1627, 14128, 5565, 4323, 7460, 13240] avg=8192.0000 max=14128 max_rank=0 min=1627 min_rank=2 imbalance_ratio=1.724609 max_load_reduction=6314 max_load_reduction_pct=0.308874 moved_to_replicas=17674 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[3811, 4843, 11621, 17947, 10527, 7640, 1454, 7693] avg=8192.0000 max=17947 max_rank=3 min=1454 min_rank=6 imbalance_ratio=2.190796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[5944, 7831, 8633, 8633, 8633, 8634, 8635, 8593] avg=8192.0000 max=8635 max_rank=6 min=5944 min_rank=0 imbalance_ratio=1.054077 max_load_reduction=9312 max_load_reduction_pct=0.518861 moved_to_replicas=27220 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[9580, 5688, 3486, 20515, 6077, 11411, 7724, 1055] avg=8192.0000 max=20515 max_rank=3 min=1055 min_rank=7 imbalance_ratio=2.504272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[14128, 5065, 1627, 14128, 5565, 4323, 7460, 13240] avg=8192.0000 max=14128 max_rank=0 min=1627 min_rank=2 imbalance_ratio=1.724609 max_load_reduction=6314 max_load_reduction_pct=0.308874 moved_to_replicas=17674 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] avg=8192.0000 max=9525 max_rank=0 min=5896 min_rank=7 imbalance_ratio=1.162720 max_load_reduction=3654 max_load_reduction_pct=0.277259 moved_to_replicas=20495 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[3811, 4843, 11621, 17947, 10527, 7640, 1454, 7693] avg=8192.0000 max=17947 max_rank=3 min=1454 min_rank=6 imbalance_ratio=2.190796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[14128, 5065, 1627, 20442, 5565, 4323, 1146, 13240] avg=8192.0000 max=20442 max_rank=3 min=1146 min_rank=6 imbalance_ratio=2.495361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[5944, 7831, 8633, 8633, 8633, 8634, 8635, 8593] avg=8192.0000 max=8635 max_rank=6 min=5944 min_rank=0 imbalance_ratio=1.054077 max_load_reduction=9312 max_load_reduction_pct=0.518861 moved_to_replicas=27220 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[8686, 8413, 7684, 8687, 6971, 8686, 7724, 8685] avg=8192.0000 max=8687 max_rank=3 min=6971 min_rank=4 imbalance_ratio=1.060425 max_load_reduction=11828 max_load_reduction_pct=0.576554 moved_to_replicas=33656 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[3811, 4843, 11621, 17947, 10527, 7640, 1454, 7693] avg=8192.0000 max=17947 max_rank=3 min=1454 min_rank=6 imbalance_ratio=2.190796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[5944, 7831, 8633, 8633, 8633, 8634, 8635, 8593] avg=8192.0000 max=8635 max_rank=6 min=5944 min_rank=0 imbalance_ratio=1.054077 max_load_reduction=9312 max_load_reduction_pct=0.518861 moved_to_replicas=27220 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[458, 6998, 15088, 9259, 2331, 17384, 10912, 3106] avg=8192.0000 max=17384 max_rank=5 min=458 min_rank=0 imbalance_ratio=2.122070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[14128, 5065, 1627, 14128, 5565, 4323, 7460, 13240] avg=8192.0000 max=14128 max_rank=0 min=1627 min_rank=2 imbalance_ratio=1.724609 max_load_reduction=6314 max_load_reduction_pct=0.308874 moved_to_replicas=17674 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[3811, 4843, 11621, 17947, 10527, 7640, 1454, 7693] avg=8192.0000 max=17947 max_rank=3 min=1454 min_rank=6 imbalance_ratio=2.190796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 1685, 3449, 19120, 2980, 9656, 12376] avg=8192.0000 max=19120 max_rank=4 min=1685 min_rank=2 imbalance_ratio=2.333984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[5944, 7831, 8633, 8633, 8633, 8634, 8635, 8593] avg=8192.0000 max=8635 max_rank=6 min=5944 min_rank=0 imbalance_ratio=1.054077 max_load_reduction=9312 max_load_reduction_pct=0.518861 moved_to_replicas=27220 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[458, 6998, 15088, 9259, 2331, 17384, 10912, 3106] avg=8192.0000 max=17384 max_rank=5 min=458 min_rank=0 imbalance_ratio=2.122070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[6930, 6998, 10911, 9259, 6508, 10912, 10912, 3106] avg=8192.0000 max=10912 max_rank=5 min=3106 min_rank=7 imbalance_ratio=1.332031 max_load_reduction=6472 max_load_reduction_pct=0.372296 moved_to_replicas=27203 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[5944, 7831, 8633, 8633, 8633, 8634, 8635, 8593] avg=8192.0000 max=8635 max_rank=6 min=5944 min_rank=0 imbalance_ratio=1.054077 max_load_reduction=9312 max_load_reduction_pct=0.518861 moved_to_replicas=27220 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[14128, 5065, 1627, 20442, 5565, 4323, 1146, 13240] avg=8192.0000 max=20442 max_rank=3 min=1146 min_rank=6 imbalance_ratio=2.495361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] avg=8192.0000 max=10404 max_rank=7 min=3449 min_rank=3 imbalance_ratio=1.270020 max_load_reduction=8716 max_load_reduction_pct=0.455858 moved_to_replicas=13521 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[458, 6998, 15088, 9259, 2331, 17384, 10912, 3106] avg=8192.0000 max=17384 max_rank=5 min=458 min_rank=0 imbalance_ratio=2.122070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[3811, 4843, 11621, 17947, 10527, 7640, 1454, 7693] avg=8192.0000 max=17947 max_rank=3 min=1454 min_rank=6 imbalance_ratio=2.190796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[6930, 6998, 10911, 9259, 6508, 10912, 10912, 3106] avg=8192.0000 max=10912 max_rank=5 min=3106 min_rank=7 imbalance_ratio=1.332031 max_load_reduction=6472 max_load_reduction_pct=0.372296 moved_to_replicas=27203 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[458, 6998, 15088, 9259, 2331, 17384, 10912, 3106] avg=8192.0000 max=17384 max_rank=5 min=458 min_rank=0 imbalance_ratio=2.122070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[14128, 5065, 1627, 14128, 5565, 4323, 7460, 13240] avg=8192.0000 max=14128 max_rank=0 min=1627 min_rank=2 imbalance_ratio=1.724609 max_load_reduction=6314 max_load_reduction_pct=0.308874 moved_to_replicas=17674 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[3471, 20547, 5550, 999, 6356, 6798, 5708, 16107] avg=8192.0000 max=20547 max_rank=1 min=999 min_rank=3 imbalance_ratio=2.508179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[6930, 6998, 10911, 9259, 6508, 10912, 10912, 3106] avg=8192.0000 max=10912 max_rank=5 min=3106 min_rank=7 imbalance_ratio=1.332031 max_load_reduction=6472 max_load_reduction_pct=0.372296 moved_to_replicas=27203 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[5944, 7831, 8633, 8633, 8633, 8634, 8635, 8593] avg=8192.0000 max=8635 max_rank=6 min=5944 min_rank=0 imbalance_ratio=1.054077 max_load_reduction=9312 max_load_reduction_pct=0.518861 moved_to_replicas=27220 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[458, 6998, 15088, 9259, 2331, 17384, 10912, 3106] avg=8192.0000 max=17384 max_rank=5 min=458 min_rank=0 imbalance_ratio=2.122070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[16295, 1555, 3419, 3477, 1223, 10803, 10160, 18604] avg=8192.0000 max=18604 max_rank=7 min=1223 min_rank=4 imbalance_ratio=2.270996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[6930, 6998, 10911, 9259, 6508, 10912, 10912, 3106] avg=8192.0000 max=10912 max_rank=5 min=3106 min_rank=7 imbalance_ratio=1.332031 max_load_reduction=6472 max_load_reduction_pct=0.372296 moved_to_replicas=27203 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[3471, 20547, 5550, 999, 6356, 6798, 5708, 16107] avg=8192.0000 max=20547 max_rank=1 min=999 min_rank=3 imbalance_ratio=2.508179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[8057, 8480, 8480, 8480, 8281, 6798, 8480, 8480] avg=8192.0000 max=8480 max_rank=1 min=6798 min_rank=5 imbalance_ratio=1.035156 max_load_reduction=12067 max_load_reduction_pct=0.587288 moved_to_replicas=30050 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[6930, 6998, 10911, 9259, 6508, 10912, 10912, 3106] avg=8192.0000 max=10912 max_rank=5 min=3106 min_rank=7 imbalance_ratio=1.332031 max_load_reduction=6472 max_load_reduction_pct=0.372296 moved_to_replicas=27203 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[3811, 4843, 11621, 17947, 10527, 7640, 1454, 7693] avg=8192.0000 max=17947 max_rank=3 min=1454 min_rank=6 imbalance_ratio=2.190796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[10804, 7046, 3419, 3477, 9024, 10803, 10160, 10803] avg=8192.0000 max=10804 max_rank=0 min=3419 min_rank=2 imbalance_ratio=1.318848 max_load_reduction=7800 max_load_reduction_pct=0.419265 moved_to_replicas=32826 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[3471, 20547, 5550, 999, 6356, 6798, 5708, 16107] avg=8192.0000 max=20547 max_rank=1 min=999 min_rank=3 imbalance_ratio=2.508179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[458, 6998, 15088, 9259, 2331, 17384, 10912, 3106] avg=8192.0000 max=17384 max_rank=5 min=458 min_rank=0 imbalance_ratio=2.122070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[8057, 8480, 8480, 8480, 8281, 6798, 8480, 8480] avg=8192.0000 max=8480 max_rank=1 min=6798 min_rank=5 imbalance_ratio=1.035156 max_load_reduction=12067 max_load_reduction_pct=0.587288 moved_to_replicas=30050 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[3471, 20547, 5550, 999, 6356, 6798, 5708, 16107] avg=8192.0000 max=20547 max_rank=1 min=999 min_rank=3 imbalance_ratio=2.508179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[5944, 7831, 8633, 8633, 8633, 8634, 8635, 8593] avg=8192.0000 max=8635 max_rank=6 min=5944 min_rank=0 imbalance_ratio=1.054077 max_load_reduction=9312 max_load_reduction_pct=0.518861 moved_to_replicas=27220 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[8057, 8480, 8480, 8480, 8281, 6798, 8480, 8480] avg=8192.0000 max=8480 max_rank=1 min=6798 min_rank=5 imbalance_ratio=1.035156 max_load_reduction=12067 max_load_reduction_pct=0.587288 moved_to_replicas=30050 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[3471, 20547, 5550, 999, 6356, 6798, 5708, 16107] avg=8192.0000 max=20547 max_rank=1 min=999 min_rank=3 imbalance_ratio=2.508179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[6930, 6998, 10911, 9259, 6508, 10912, 10912, 3106] avg=8192.0000 max=10912 max_rank=5 min=3106 min_rank=7 imbalance_ratio=1.332031 max_load_reduction=6472 max_load_reduction_pct=0.372296 moved_to_replicas=27203 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[8057, 8480, 8480, 8480, 8281, 6798, 8480, 8480] avg=8192.0000 max=8480 max_rank=1 min=6798 min_rank=5 imbalance_ratio=1.035156 max_load_reduction=12067 max_load_reduction_pct=0.587288 moved_to_replicas=30050 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[14357, 5595, 16727, 6706, 8722, 1704, 939, 10786] avg=8192.0000 max=16727 max_rank=2 min=939 min_rank=6 imbalance_ratio=2.041870 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[8057, 8480, 8480, 8480, 8281, 6798, 8480, 8480] avg=8192.0000 max=8480 max_rank=1 min=6798 min_rank=5 imbalance_ratio=1.035156 max_load_reduction=12067 max_load_reduction_pct=0.587288 moved_to_replicas=30050 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[458, 6998, 15088, 9259, 2331, 17384, 10912, 3106] avg=8192.0000 max=17384 max_rank=5 min=458 min_rank=0 imbalance_ratio=2.122070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[3471, 20547, 5550, 999, 6356, 6798, 5708, 16107] avg=8192.0000 max=20547 max_rank=1 min=999 min_rank=3 imbalance_ratio=2.508179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[10786, 5595, 10785, 6706, 8722, 5275, 6881, 10786] avg=8192.0000 max=10786 max_rank=0 min=5275 min_rank=5 imbalance_ratio=1.316650 max_load_reduction=5941 max_load_reduction_pct=0.355174 moved_to_replicas=21432 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[6930, 6998, 10911, 9259, 6508, 10912, 10912, 3106] avg=8192.0000 max=10912 max_rank=5 min=3106 min_rank=7 imbalance_ratio=1.332031 max_load_reduction=6472 max_load_reduction_pct=0.372296 moved_to_replicas=27203 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[8057, 8480, 8480, 8480, 8281, 6798, 8480, 8480] avg=8192.0000 max=8480 max_rank=1 min=6798 min_rank=5 imbalance_ratio=1.035156 max_load_reduction=12067 max_load_reduction_pct=0.587288 moved_to_replicas=30050 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[7488, 3101, 7858, 15863, 6603, 1919, 7446, 15258] avg=8192.0000 max=15863 max_rank=3 min=1919 min_rank=5 imbalance_ratio=1.936401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[3471, 20547, 5550, 999, 6356, 6798, 5708, 16107] avg=8192.0000 max=20547 max_rank=1 min=999 min_rank=3 imbalance_ratio=2.508179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[7488, 9179, 7858, 9181, 6603, 8601, 7446, 9180] avg=8192.0000 max=9181 max_rank=3 min=6603 min_rank=4 imbalance_ratio=1.120728 max_load_reduction=6682 max_load_reduction_pct=0.421232 moved_to_replicas=18181 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[8057, 8480, 8480, 8480, 8281, 6798, 8480, 8480] avg=8192.0000 max=8480 max_rank=1 min=6798 min_rank=5 imbalance_ratio=1.035156 max_load_reduction=12067 max_load_reduction_pct=0.587288 moved_to_replicas=30050 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[16315, 21035, 925, 1371, 5032, 3582, 9930, 7346] avg=8192.0000 max=21035 max_rank=1 min=925 min_rank=2 imbalance_ratio=2.567749 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[8695, 8696, 8697, 5938, 7538, 8696, 8695, 8581] avg=8192.0000 max=8697 max_rank=2 min=5938 min_rank=3 imbalance_ratio=1.061646 max_load_reduction=12338 max_load_reduction_pct=0.586546 moved_to_replicas=28653 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[22351, 8114, 4267, 7165, 1125, 2738, 1983, 17793] avg=8192.0000 max=22351 max_rank=0 min=1125 min_rank=4 imbalance_ratio=2.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=283999 total_dropped_tokens=5105 drop_rate_pct=1.797542 records_with_drop_pct=15.625000 peak_rank_overflow=3888 total_slack_slots=46720 peak_demand=14128 required_capacity=1.724609 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=32 total_expected_recv=256941 total_dropped_tokens=1965 drop_rate_pct=0.764767 records_with_drop_pct=6.250000 peak_rank_overflow=1037 total_slack_slots=69888 peak_demand=11277 required_capacity=1.376587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[8498, 8114, 7801, 7165, 8499, 8497, 8462, 8500] avg=8192.0000 max=8500 max_rank=7 min=7165 min_rank=3 imbalance_ratio=1.037598 max_load_reduction=13851 max_load_reduction_pct=0.619704 moved_to_replicas=38887 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=32 total_expected_recv=258238 total_dropped_tokens=1023 drop_rate_pct=0.396146 records_with_drop_pct=6.250000 peak_rank_overflow=861 total_slack_slots=67840 peak_demand=11101 required_capacity=1.355103 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=32 total_expected_recv=277280 total_dropped_tokens=5134 drop_rate_pct=1.851558 records_with_drop_pct=15.625000 peak_rank_overflow=3000 total_slack_slots=52992 peak_demand=13240 required_capacity=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=32 total_expected_recv=262076 total_dropped_tokens=1600 drop_rate_pct=0.610510 records_with_drop_pct=6.250000 peak_rank_overflow=928 total_slack_slots=64640 peak_demand=11168 required_capacity=1.363281 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[9580, 5688, 3486, 20515, 6077, 11411, 7724, 1055] avg=8192.0000 max=20515 max_rank=3 min=1055 min_rank=7 imbalance_ratio=2.504272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[8686, 8413, 7684, 8687, 6971, 8686, 7724, 8685] avg=8192.0000 max=8687 max_rank=3 min=6971 min_rank=4 imbalance_ratio=1.060425 max_load_reduction=11828 max_load_reduction_pct=0.576554 moved_to_replicas=33656 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=32 total_expected_recv=255789 total_dropped_tokens=1235 drop_rate_pct=0.482820 records_with_drop_pct=6.250000 peak_rank_overflow=672 total_slack_slots=70656 peak_demand=10912 required_capacity=1.332031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[14128, 5065, 1627, 20442, 5565, 4323, 1146, 13240] avg=8192.0000 max=20442 max_rank=3 min=1146 min_rank=6 imbalance_ratio=2.495361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[14128, 5065, 1627, 14128, 5565, 4323, 7460, 13240] avg=8192.0000 max=14128 max_rank=0 min=1627 min_rank=2 imbalance_ratio=1.724609 max_load_reduction=6314 max_load_reduction_pct=0.308874 moved_to_replicas=17674 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=32 total_expected_recv=242624 total_dropped_tokens=1379 drop_rate_pct=0.568369 records_with_drop_pct=9.375000 peak_rank_overflow=671 total_slack_slots=83456 peak_demand=10911 required_capacity=1.331909 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[3811, 4843, 11621, 17947, 10527, 7640, 1454, 7693] avg=8192.0000 max=17947 max_rank=3 min=1454 min_rank=6 imbalance_ratio=2.190796 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[5944, 7831, 8633, 8633, 8633, 8634, 8635, 8593] avg=8192.0000 max=8635 max_rank=6 min=5944 min_rank=0 imbalance_ratio=1.054077 max_load_reduction=9312 max_load_reduction_pct=0.518861 moved_to_replicas=27220 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[458, 6998, 15088, 9259, 2331, 17384, 10912, 3106] avg=8192.0000 max=17384 max_rank=5 min=458 min_rank=0 imbalance_ratio=2.122070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[6930, 6998, 10911, 9259, 6508, 10912, 10912, 3106] avg=8192.0000 max=10912 max_rank=5 min=3106 min_rank=7 imbalance_ratio=1.332031 max_load_reduction=6472 max_load_reduction_pct=0.372296 moved_to_replicas=27203 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[3471, 20547, 5550, 999, 6356, 6798, 5708, 16107] avg=8192.0000 max=20547 max_rank=1 min=999 min_rank=3 imbalance_ratio=2.508179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[8057, 8480, 8480, 8480, 8281, 6798, 8480, 8480] avg=8192.0000 max=8480 max_rank=1 min=6798 min_rank=5 imbalance_ratio=1.035156 max_load_reduction=12067 max_load_reduction_pct=0.587288 moved_to_replicas=30050 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=32 total_expected_recv=260205 total_dropped_tokens=4926 drop_rate_pct=1.893123 records_with_drop_pct=6.250000 peak_rank_overflow=3888 total_slack_slots=69888 peak_demand=14128 required_capacity=1.724609 + [2026-08-30 05:07:56] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 28440.7 | throughput per GPU (TFLOP/s/GPU): 11.7 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.104432E+01 | loss scale: 1.0 | grad norm: 35.704 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 10240 | min safe capacity: 1.2500 | +Number of parameters in transformer block in billions: 23.22 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 58332.30029296875 | max allocated: 74609.11376953125 | reserved: 80984.0 | max reserved: 80984.0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=10240 expected_recv=17323 expected_overflow=0 exact_recv_per_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=10240 expected_recv=17322 expected_overflow=0 exact_recv_per_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=10240 expected_recv=15914 expected_overflow=0 exact_recv_per_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=10240 expected_recv=17321 expected_overflow=0 exact_recv_per_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=10240 expected_recv=17322 expected_overflow=0 exact_recv_per_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=10240 expected_recv=15261 expected_overflow=0 exact_recv_per_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=10240 expected_recv=13288 expected_overflow=0 exact_recv_per_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=10240 expected_recv=17323 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=10240 expected_recv=17322 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=10240 expected_recv=17321 expected_overflow=0 exact_recv_per_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=10240 expected_recv=15914 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=10240 expected_recv=17321 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=10240 expected_recv=17322 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17536 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=10240 expected_recv=15261 actual_valid_recv=15616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=10240 expected_recv=13288 actual_valid_recv=13568 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=15616 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=13568 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=10240 expected_recv=17321 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=10240 expected_recv=18812 expected_overflow=670 exact_recv_per_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=10240 expected_recv=16960 expected_overflow=0 exact_recv_per_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=10240 expected_recv=18812 actual_valid_recv=18176 observed_loss=670 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=10240 expected_recv=19603 expected_overflow=289 exact_recv_per_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=10240 expected_recv=8676 expected_overflow=0 exact_recv_per_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=10240 expected_recv=15429 expected_overflow=0 exact_recv_per_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=10240 expected_recv=16960 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=10240 expected_recv=19930 expected_overflow=615 exact_recv_per_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=18176 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=10240 expected_recv=19603 actual_valid_recv=19328 observed_loss=289 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=10240 expected_recv=11677 expected_overflow=0 exact_recv_per_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=17152 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=10240 expected_recv=8676 actual_valid_recv=8832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=10240 expected_recv=15429 actual_valid_recv=15744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=10240 expected_recv=19930 actual_valid_recv=19328 observed_loss=615 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=19328 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=10240 expected_recv=11677 actual_valid_recv=11904 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=8832 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=15744 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=19328 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=10240 expected_recv=19985 expected_overflow=671 exact_recv_per_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=11904 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=10240 expected_recv=19985 actual_valid_recv=19328 observed_loss=671 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=10240 expected_recv=19429 expected_overflow=0 exact_recv_per_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=19328 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=10240 expected_recv=19429 actual_valid_recv=19456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=10240 expected_recv=9188 expected_overflow=0 exact_recv_per_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=19456 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=10240 expected_recv=9188 actual_valid_recv=9344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=10240 expected_recv=13211 expected_overflow=0 exact_recv_per_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=10240 expected_recv=17816 expected_overflow=0 exact_recv_per_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=10240 expected_recv=18652 expected_overflow=0 exact_recv_per_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=10240 expected_recv=16707 expected_overflow=0 exact_recv_per_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=10240 expected_recv=19428 expected_overflow=0 exact_recv_per_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=9344 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=10240 expected_recv=13211 actual_valid_recv=13568 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=10240 expected_recv=17816 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=10240 expected_recv=18652 actual_valid_recv=18688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=10240 expected_recv=16707 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=10240 expected_recv=19428 actual_valid_recv=19456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=10240 expected_recv=16641 expected_overflow=0 exact_recv_per_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=13568 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=17920 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=18688 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=10240 expected_recv=15534 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=19456 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=10240 expected_recv=16641 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=10240 expected_recv=15534 actual_valid_recv=15872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=10240 expected_recv=17046 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=15872 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=10240 expected_recv=17046 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=10240 expected_recv=17046 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=10240 expected_recv=17046 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=10240 expected_recv=17045 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=10240 expected_recv=14576 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=10240 expected_recv=16789 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=10240 expected_recv=17046 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=10240 expected_recv=15990 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=10240 expected_recv=17046 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=10240 expected_recv=16215 expected_overflow=0 exact_recv_per_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17152 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=10240 expected_recv=17045 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=10240 expected_recv=14576 actual_valid_recv=14720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=10240 expected_recv=16789 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=10240 expected_recv=15990 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17280 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=10240 expected_recv=16215 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=10240 expected_recv=16290 expected_overflow=0 exact_recv_per_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17280 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=14720 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=10240 expected_recv=16290 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=10240 expected_recv=15594 expected_overflow=0 exact_recv_per_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=10240 expected_recv=16680 expected_overflow=0 exact_recv_per_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=10240 expected_recv=15594 actual_valid_recv=15872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=10240 expected_recv=16679 expected_overflow=0 exact_recv_per_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=10240 expected_recv=16388 expected_overflow=0 exact_recv_per_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=10240 expected_recv=16680 expected_overflow=0 exact_recv_per_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=10240 expected_recv=16680 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=15872 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=10240 expected_recv=16265 expected_overflow=0 exact_recv_per_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=10240 expected_recv=16679 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=10240 expected_recv=16388 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=10240 expected_recv=16680 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=10240 expected_recv=16265 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=10240 expected_recv=16023 expected_overflow=0 exact_recv_per_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=10240 expected_recv=16023 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=10240 expected_recv=16675 expected_overflow=0 exact_recv_per_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=10240 expected_recv=16675 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=10240 expected_recv=16916 expected_overflow=0 exact_recv_per_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=10240 expected_recv=16916 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=10240 expected_recv=16914 expected_overflow=0 exact_recv_per_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=10240 expected_recv=7340 expected_overflow=0 exact_recv_per_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=10240 expected_recv=15020 expected_overflow=0 exact_recv_per_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=17280 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=10240 expected_recv=16345 expected_overflow=0 exact_recv_per_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=10240 expected_recv=16914 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=10240 expected_recv=16914 expected_overflow=0 exact_recv_per_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=10240 expected_recv=7340 actual_valid_recv=7424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=10240 expected_recv=15020 actual_valid_recv=15232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=10240 expected_recv=20665 expected_overflow=796 exact_recv_per_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=10240 expected_recv=16345 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=7424 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=10240 expected_recv=16914 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=15232 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=10240 expected_recv=20665 actual_valid_recv=19968 observed_loss=796 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=10240 expected_recv=12621 expected_overflow=0 exact_recv_per_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=10240 expected_recv=17772 expected_overflow=0 exact_recv_per_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=19968 slack_slots=512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=10240 expected_recv=12621 actual_valid_recv=12800 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=10240 expected_recv=17772 actual_valid_recv=18048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=12800 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=18048 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=10240 expected_recv=16684 expected_overflow=0 exact_recv_per_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=10240 expected_recv=15032 expected_overflow=0 exact_recv_per_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=10240 expected_recv=16684 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=10240 expected_recv=20665 expected_overflow=796 exact_recv_per_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=10240 expected_recv=20667 expected_overflow=797 exact_recv_per_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=10240 expected_recv=14658 expected_overflow=0 exact_recv_per_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=10240 expected_recv=15032 actual_valid_recv=15232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=10240 expected_recv=17750 expected_overflow=0 exact_recv_per_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=10240 expected_recv=20665 actual_valid_recv=19968 observed_loss=796 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=10240 expected_recv=14658 actual_valid_recv=14976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=15232 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=10240 expected_recv=20667 actual_valid_recv=19968 observed_loss=797 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=10240 expected_recv=14978 expected_overflow=0 exact_recv_per_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=19968 slack_slots=512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=10240 expected_recv=17750 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=14976 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=19968 slack_slots=512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=10240 expected_recv=18723 expected_overflow=0 exact_recv_per_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=10240 expected_recv=14978 actual_valid_recv=15232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=17920 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=10240 expected_recv=18723 actual_valid_recv=18816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=15232 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=10240 expected_recv=13623 expected_overflow=0 exact_recv_per_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=18816 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=10240 expected_recv=21680 expected_overflow=2173 exact_recv_per_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=10240 expected_recv=13623 actual_valid_recv=13824 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=10240 expected_recv=15779 expected_overflow=0 exact_recv_per_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=10240 expected_recv=17839 expected_overflow=0 exact_recv_per_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=10240 expected_recv=21680 actual_valid_recv=19584 observed_loss=2173 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=13824 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=10240 expected_recv=17348 expected_overflow=0 exact_recv_per_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=10240 expected_recv=15779 actual_valid_recv=16000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=10240 expected_recv=16977 expected_overflow=578 exact_recv_per_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=10240 expected_recv=17839 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=19584 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=10240 expected_recv=9363 expected_overflow=0 exact_recv_per_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16000 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=10240 expected_recv=17348 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=17920 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=10240 expected_recv=16977 actual_valid_recv=16512 observed_loss=578 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=10240 expected_recv=9363 actual_valid_recv=9600 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=10240 expected_recv=10746 expected_overflow=0 exact_recv_per_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=9600 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=10240 expected_recv=10746 actual_valid_recv=10880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=10240 expected_recv=19535 expected_overflow=26 exact_recv_per_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=10240 expected_recv=15211 expected_overflow=0 exact_recv_per_rank=[9023, 9763, 9578, 6296, 9764, 3654, 9764, 7694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=10880 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=10240 expected_recv=19535 actual_valid_recv=19584 observed_loss=26 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=10240 expected_recv=21682 expected_overflow=2173 exact_recv_per_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=10240 expected_recv=11953 expected_overflow=0 exact_recv_per_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=10240 expected_recv=15211 actual_valid_recv=15488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=19584 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=10240 expected_recv=19136 expected_overflow=0 exact_recv_per_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=15488 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=10240 expected_recv=21682 actual_valid_recv=19584 observed_loss=2173 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=10240 expected_recv=11953 actual_valid_recv=12160 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=10240 expected_recv=13376 expected_overflow=0 exact_recv_per_rank=[9023, 9763, 9578, 6296, 9764, 3654, 9764, 7694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=10240 expected_recv=19136 actual_valid_recv=19328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=19584 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=12160 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=10240 expected_recv=13376 actual_valid_recv=13568 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=19328 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=10240 expected_recv=16806 expected_overflow=0 exact_recv_per_rank=[9023, 9763, 9578, 6296, 9764, 3654, 9764, 7694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=10240 expected_recv=18734 expected_overflow=0 exact_recv_per_rank=[9023, 9763, 9578, 6296, 9764, 3654, 9764, 7694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=13568 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=10240 expected_recv=16806 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=10240 expected_recv=12622 expected_overflow=0 exact_recv_per_rank=[9023, 9763, 9578, 6296, 9764, 3654, 9764, 7694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=10240 expected_recv=18734 actual_valid_recv=18944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=10240 expected_recv=17957 expected_overflow=0 exact_recv_per_rank=[8497, 8497, 8497, 8104, 8497, 7967, 6980, 8497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=10240 expected_recv=12622 actual_valid_recv=12800 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=18944 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=10240 expected_recv=17957 actual_valid_recv=18048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=10240 expected_recv=18734 expected_overflow=0 exact_recv_per_rank=[9023, 9763, 9578, 6296, 9764, 3654, 9764, 7694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=10240 expected_recv=17596 expected_overflow=0 exact_recv_per_rank=[9023, 9763, 9578, 6296, 9764, 3654, 9764, 7694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=12800 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=18048 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=10240 expected_recv=17993 expected_overflow=0 exact_recv_per_rank=[9023, 9763, 9578, 6296, 9764, 3654, 9764, 7694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=10240 expected_recv=18349 expected_overflow=0 exact_recv_per_rank=[8497, 8497, 8497, 8104, 8497, 7967, 6980, 8497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=10240 expected_recv=18734 actual_valid_recv=18944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=10240 expected_recv=17596 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=10240 expected_recv=17993 actual_valid_recv=18176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=10240 expected_recv=14371 expected_overflow=0 exact_recv_per_rank=[8497, 8497, 8497, 8104, 8497, 7967, 6980, 8497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=18944 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=10240 expected_recv=18349 actual_valid_recv=18432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=17792 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=18176 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=10240 expected_recv=18349 expected_overflow=0 exact_recv_per_rank=[8497, 8497, 8497, 8104, 8497, 7967, 6980, 8497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=18432 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=10240 expected_recv=14371 actual_valid_recv=14720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=10240 expected_recv=18349 actual_valid_recv=18432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=14720 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=10240 expected_recv=17819 expected_overflow=0 exact_recv_per_rank=[8497, 8497, 8497, 8104, 8497, 7967, 6980, 8497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=10240 expected_recv=19172 expected_overflow=0 exact_recv_per_rank=[9486, 7846, 4806, 9484, 5659, 9485, 9285, 9485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=18432 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=10240 expected_recv=17819 actual_valid_recv=18048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=10240 expected_recv=19172 actual_valid_recv=19328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=10240 expected_recv=12041 expected_overflow=0 exact_recv_per_rank=[8497, 8497, 8497, 8104, 8497, 7967, 6980, 8497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=10240 expected_recv=13838 expected_overflow=0 exact_recv_per_rank=[8497, 8497, 8497, 8104, 8497, 7967, 6980, 8497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=18048 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=19328 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=10240 expected_recv=18348 expected_overflow=0 exact_recv_per_rank=[8497, 8497, 8497, 8104, 8497, 7967, 6980, 8497] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=10240 expected_recv=14244 expected_overflow=0 exact_recv_per_rank=[9486, 7846, 4806, 9484, 5659, 9485, 9285, 9485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=10240 expected_recv=12041 actual_valid_recv=12288 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=10240 expected_recv=13838 actual_valid_recv=14080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=10240 expected_recv=18348 actual_valid_recv=18432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=12288 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=10240 expected_recv=14244 actual_valid_recv=14464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=14080 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=10240 expected_recv=14493 expected_overflow=0 exact_recv_per_rank=[9486, 7846, 4806, 9484, 5659, 9485, 9285, 9485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=18432 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=10240 expected_recv=11580 expected_overflow=0 exact_recv_per_rank=[9486, 7846, 4806, 9484, 5659, 9485, 9285, 9485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=14464 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=10240 expected_recv=14493 actual_valid_recv=14720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=10240 expected_recv=11580 actual_valid_recv=11904 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=14720 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=10240 expected_recv=18967 expected_overflow=132 exact_recv_per_rank=[9230, 10373, 4882, 10372, 5185, 4754, 10373, 10367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=10240 expected_recv=18136 expected_overflow=0 exact_recv_per_rank=[9486, 7846, 4806, 9484, 5659, 9485, 9285, 9485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=11904 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=10240 expected_recv=18967 actual_valid_recv=18944 observed_loss=132 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=10240 expected_recv=18136 actual_valid_recv=18304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=10240 expected_recv=18927 expected_overflow=0 exact_recv_per_rank=[9486, 7846, 4806, 9484, 5659, 9485, 9285, 9485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=10240 expected_recv=15347 expected_overflow=0 exact_recv_per_rank=[9486, 7846, 4806, 9484, 5659, 9485, 9285, 9485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=18944 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=10240 expected_recv=19173 expected_overflow=0 exact_recv_per_rank=[9486, 7846, 4806, 9484, 5659, 9485, 9285, 9485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=18304 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=10240 expected_recv=18927 actual_valid_recv=19072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=10240 expected_recv=18321 expected_overflow=127 exact_recv_per_rank=[9230, 10373, 4882, 10372, 5185, 4754, 10373, 10367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=10240 expected_recv=15347 actual_valid_recv=15488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=10240 expected_recv=19173 actual_valid_recv=19328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=19072 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=15488 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=10240 expected_recv=18321 actual_valid_recv=18304 observed_loss=127 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=10240 expected_recv=13386 expected_overflow=0 exact_recv_per_rank=[9230, 10373, 4882, 10372, 5185, 4754, 10373, 10367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=19328 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=10240 expected_recv=18966 expected_overflow=133 exact_recv_per_rank=[9230, 10373, 4882, 10372, 5185, 4754, 10373, 10367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=18304 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=10240 expected_recv=13386 actual_valid_recv=13568 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=10240 expected_recv=18966 actual_valid_recv=18944 observed_loss=133 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=10240 expected_recv=22164 expected_overflow=2484 exact_recv_per_rank=[9442, 5698, 9441, 9440, 9441, 5228, 7405, 9441] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=13568 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=10240 expected_recv=13349 expected_overflow=0 exact_recv_per_rank=[9230, 10373, 4882, 10372, 5185, 4754, 10373, 10367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=18944 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=10240 expected_recv=22164 actual_valid_recv=19712 observed_loss=2484 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=10240 expected_recv=13349 actual_valid_recv=13568 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=10240 expected_recv=13394 expected_overflow=0 exact_recv_per_rank=[9230, 10373, 4882, 10372, 5185, 4754, 10373, 10367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=10240 expected_recv=16865 expected_overflow=133 exact_recv_per_rank=[9230, 10373, 4882, 10372, 5185, 4754, 10373, 10367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=19712 slack_slots=768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=13568 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=10240 expected_recv=13394 actual_valid_recv=13568 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=10240 expected_recv=16865 actual_valid_recv=16896 observed_loss=133 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=10240 expected_recv=17824 expected_overflow=0 exact_recv_per_rank=[9230, 10373, 4882, 10372, 5185, 4754, 10373, 10367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=10240 expected_recv=20545 expected_overflow=864 exact_recv_per_rank=[9442, 5698, 9441, 9440, 9441, 5228, 7405, 9441] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=13568 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=10240 expected_recv=20545 actual_valid_recv=19712 observed_loss=864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=10240 expected_recv=17824 actual_valid_recv=18048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=19712 slack_slots=768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=18048 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=10240 expected_recv=9309 expected_overflow=0 exact_recv_per_rank=[9442, 5698, 9441, 9440, 9441, 5228, 7405, 9441] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=10240 expected_recv=17611 expected_overflow=0 exact_recv_per_rank=[9316, 7294, 7362, 9318, 5885, 9319, 7724, 9318] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=10240 expected_recv=19754 expected_overflow=73 exact_recv_per_rank=[9442, 5698, 9441, 9440, 9441, 5228, 7405, 9441] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=10240 expected_recv=9309 actual_valid_recv=9600 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=10240 expected_recv=12346 expected_overflow=0 exact_recv_per_rank=[9442, 5698, 9441, 9440, 9441, 5228, 7405, 9441] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=10240 expected_recv=17611 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=9600 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=10240 expected_recv=19754 actual_valid_recv=19712 observed_loss=73 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=17664 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=10240 expected_recv=12346 actual_valid_recv=12672 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=10240 expected_recv=22165 expected_overflow=2484 exact_recv_per_rank=[9442, 5698, 9441, 9440, 9441, 5228, 7405, 9441] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=19712 slack_slots=768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=10240 expected_recv=9698 expected_overflow=0 exact_recv_per_rank=[9442, 5698, 9441, 9440, 9441, 5228, 7405, 9441] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=12672 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=10240 expected_recv=18099 expected_overflow=0 exact_recv_per_rank=[9316, 7294, 7362, 9318, 5885, 9319, 7724, 9318] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=10240 expected_recv=15091 expected_overflow=0 exact_recv_per_rank=[9442, 5698, 9441, 9440, 9441, 5228, 7405, 9441] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=10240 expected_recv=22165 actual_valid_recv=19712 observed_loss=2484 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=10240 expected_recv=9698 actual_valid_recv=9856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=10240 expected_recv=18099 actual_valid_recv=18176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=10240 expected_recv=15091 actual_valid_recv=15360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=19712 slack_slots=768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=9856 slack_slots=10624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=18176 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=15360 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=10240 expected_recv=11057 expected_overflow=0 exact_recv_per_rank=[5744, 10854, 6322, 7859, 5277, 10856, 7770, 10854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=10240 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[9316, 7294, 7362, 9318, 5885, 9319, 7724, 9318] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=10240 expected_recv=11057 actual_valid_recv=11392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=10240 expected_recv=15500 expected_overflow=0 exact_recv_per_rank=[9316, 7294, 7362, 9318, 5885, 9319, 7724, 9318] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=10240 expected_recv=15888 actual_valid_recv=16000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=11392 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16000 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=10240 expected_recv=15500 actual_valid_recv=15744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=15744 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=10240 expected_recv=14665 expected_overflow=0 exact_recv_per_rank=[9316, 7294, 7362, 9318, 5885, 9319, 7724, 9318] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=10240 expected_recv=17315 expected_overflow=614 exact_recv_per_rank=[5744, 10854, 6322, 7859, 5277, 10856, 7770, 10854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=10240 expected_recv=16504 expected_overflow=0 exact_recv_per_rank=[9316, 7294, 7362, 9318, 5885, 9319, 7724, 9318] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=10240 expected_recv=14706 expected_overflow=0 exact_recv_per_rank=[9316, 7294, 7362, 9318, 5885, 9319, 7724, 9318] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=10240 expected_recv=18099 expected_overflow=0 exact_recv_per_rank=[9316, 7294, 7362, 9318, 5885, 9319, 7724, 9318] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=10240 expected_recv=14665 actual_valid_recv=14976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=10240 expected_recv=17315 actual_valid_recv=16768 observed_loss=614 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=10240 expected_recv=16504 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=10240 expected_recv=14706 actual_valid_recv=14976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=10240 expected_recv=18099 actual_valid_recv=18176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=14976 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=14976 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=18176 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=10240 expected_recv=21946 expected_overflow=1466 exact_recv_per_rank=[5744, 10854, 6322, 7859, 5277, 10856, 7770, 10854] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[880, 26566, 4616, 2953, 11158, 3239, 6042, 10082] avg=8192.0000 max=26566 max_rank=1 min=880 min_rank=0 imbalance_ratio=3.242920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=10240 expected_recv=11490 expected_overflow=0 exact_recv_per_rank=[5744, 10854, 6322, 7859, 5277, 10856, 7770, 10854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=10240 expected_recv=21946 actual_valid_recv=20480 observed_loss=1466 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=10240 expected_recv=11490 actual_valid_recv=11648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=20480 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=11648 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] avg=8192.0000 max=8541 max_rank=0 min=7234 min_rank=2 imbalance_ratio=1.042603 max_load_reduction=18025 max_load_reduction_pct=0.678499 moved_to_replicas=43107 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=10240 expected_recv=13481 expected_overflow=0 exact_recv_per_rank=[5744, 10854, 6322, 7859, 5277, 10856, 7770, 10854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=10240 expected_recv=17001 expected_overflow=0 exact_recv_per_rank=[5744, 10854, 6322, 7859, 5277, 10856, 7770, 10854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=10240 expected_recv=16834 expected_overflow=850 exact_recv_per_rank=[5744, 10854, 6322, 7859, 5277, 10856, 7770, 10854] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[880, 26566, 4616, 2953, 11158, 3239, 6042, 10082] avg=8192.0000 max=26566 max_rank=1 min=880 min_rank=0 imbalance_ratio=3.242920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=10240 expected_recv=21948 expected_overflow=1468 exact_recv_per_rank=[5744, 10854, 6322, 7859, 5277, 10856, 7770, 10854] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=10240 expected_recv=13481 actual_valid_recv=13696 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=10240 expected_recv=17001 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=10240 expected_recv=16834 actual_valid_recv=16000 observed_loss=850 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=10240 expected_recv=21948 actual_valid_recv=20480 observed_loss=1468 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=13696 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=17280 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16000 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=20480 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] avg=8192.0000 max=8541 max_rank=0 min=7234 min_rank=2 imbalance_ratio=1.042603 max_load_reduction=18025 max_load_reduction_pct=0.678499 moved_to_replicas=43107 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[880, 26566, 4616, 2953, 11158, 3239, 6042, 10082] avg=8192.0000 max=26566 max_rank=1 min=880 min_rank=0 imbalance_ratio=3.242920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[880, 26566, 4616, 2953, 11158, 3239, 6042, 10082] avg=8192.0000 max=26566 max_rank=1 min=880 min_rank=0 imbalance_ratio=3.242920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[8102, 1081, 2303, 7902, 17066, 4358, 11235, 13489] avg=8192.0000 max=17066 max_rank=4 min=1081 min_rank=1 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[880, 26566, 4616, 2953, 11158, 3239, 6042, 10082] avg=8192.0000 max=26566 max_rank=1 min=880 min_rank=0 imbalance_ratio=3.242920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[880, 26566, 4616, 2953, 11158, 3239, 6042, 10082] avg=8192.0000 max=26566 max_rank=1 min=880 min_rank=0 imbalance_ratio=3.242920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] avg=8192.0000 max=8541 max_rank=0 min=7234 min_rank=2 imbalance_ratio=1.042603 max_load_reduction=18025 max_load_reduction_pct=0.678499 moved_to_replicas=43107 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[880, 26566, 4616, 2953, 11158, 3239, 6042, 10082] avg=8192.0000 max=26566 max_rank=1 min=880 min_rank=0 imbalance_ratio=3.242920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] avg=8192.0000 max=8541 max_rank=0 min=7234 min_rank=2 imbalance_ratio=1.042603 max_load_reduction=18025 max_load_reduction_pct=0.678499 moved_to_replicas=43107 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[880, 26566, 4616, 2953, 11158, 3239, 6042, 10082] avg=8192.0000 max=26566 max_rank=1 min=880 min_rank=0 imbalance_ratio=3.242920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[8102, 1081, 2303, 7902, 17066, 4358, 11235, 13489] avg=8192.0000 max=17066 max_rank=4 min=1081 min_rank=1 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] avg=8192.0000 max=9075 max_rank=6 min=6518 min_rank=5 imbalance_ratio=1.107788 max_load_reduction=7991 max_load_reduction_pct=0.468241 moved_to_replicas=19465 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] avg=8192.0000 max=8541 max_rank=0 min=7234 min_rank=2 imbalance_ratio=1.042603 max_load_reduction=18025 max_load_reduction_pct=0.678499 moved_to_replicas=43107 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] avg=8192.0000 max=8541 max_rank=0 min=7234 min_rank=2 imbalance_ratio=1.042603 max_load_reduction=18025 max_load_reduction_pct=0.678499 moved_to_replicas=43107 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] avg=8192.0000 max=8541 max_rank=0 min=7234 min_rank=2 imbalance_ratio=1.042603 max_load_reduction=18025 max_load_reduction_pct=0.678499 moved_to_replicas=43107 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] avg=8192.0000 max=8541 max_rank=0 min=7234 min_rank=2 imbalance_ratio=1.042603 max_load_reduction=18025 max_load_reduction_pct=0.678499 moved_to_replicas=43107 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] avg=8192.0000 max=9075 max_rank=6 min=6518 min_rank=5 imbalance_ratio=1.107788 max_load_reduction=7991 max_load_reduction_pct=0.468241 moved_to_replicas=19465 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[8102, 1081, 2303, 7902, 17066, 4358, 11235, 13489] avg=8192.0000 max=17066 max_rank=4 min=1081 min_rank=1 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[8102, 1081, 2303, 7902, 17066, 4358, 11235, 13489] avg=8192.0000 max=17066 max_rank=4 min=1081 min_rank=1 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[11173, 1843, 8958, 17066, 1359, 12417, 8436, 4284] avg=8192.0000 max=17066 max_rank=3 min=1359 min_rank=4 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[8102, 1081, 2303, 7902, 17066, 4358, 11235, 13489] avg=8192.0000 max=17066 max_rank=4 min=1081 min_rank=1 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] avg=8192.0000 max=9075 max_rank=6 min=6518 min_rank=5 imbalance_ratio=1.107788 max_load_reduction=7991 max_load_reduction_pct=0.468241 moved_to_replicas=19465 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[8102, 1081, 2303, 7902, 17066, 4358, 11235, 13489] avg=8192.0000 max=17066 max_rank=4 min=1081 min_rank=1 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] avg=8192.0000 max=9075 max_rank=6 min=6518 min_rank=5 imbalance_ratio=1.107788 max_load_reduction=7991 max_load_reduction_pct=0.468241 moved_to_replicas=19465 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[8102, 1081, 2303, 7902, 17066, 4358, 11235, 13489] avg=8192.0000 max=17066 max_rank=4 min=1081 min_rank=1 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[11173, 1843, 8958, 17066, 1359, 12417, 8436, 4284] avg=8192.0000 max=17066 max_rank=3 min=1359 min_rank=4 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] avg=8192.0000 max=9213 max_rank=0 min=5048 min_rank=1 imbalance_ratio=1.124634 max_load_reduction=7853 max_load_reduction_pct=0.460155 moved_to_replicas=24564 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[8102, 1081, 2303, 7902, 17066, 4358, 11235, 13489] avg=8192.0000 max=17066 max_rank=4 min=1081 min_rank=1 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] avg=8192.0000 max=9075 max_rank=6 min=6518 min_rank=5 imbalance_ratio=1.107788 max_load_reduction=7991 max_load_reduction_pct=0.468241 moved_to_replicas=19465 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] avg=8192.0000 max=9075 max_rank=6 min=6518 min_rank=5 imbalance_ratio=1.107788 max_load_reduction=7991 max_load_reduction_pct=0.468241 moved_to_replicas=19465 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] avg=8192.0000 max=9075 max_rank=6 min=6518 min_rank=5 imbalance_ratio=1.107788 max_load_reduction=7991 max_load_reduction_pct=0.468241 moved_to_replicas=19465 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] avg=8192.0000 max=9213 max_rank=0 min=5048 min_rank=1 imbalance_ratio=1.124634 max_load_reduction=7853 max_load_reduction_pct=0.460155 moved_to_replicas=24564 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[11173, 1843, 8958, 17066, 1359, 12417, 8436, 4284] avg=8192.0000 max=17066 max_rank=3 min=1359 min_rank=4 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] avg=8192.0000 max=9075 max_rank=6 min=6518 min_rank=5 imbalance_ratio=1.107788 max_load_reduction=7991 max_load_reduction_pct=0.468241 moved_to_replicas=19465 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[11173, 1843, 8958, 17066, 1359, 12417, 8436, 4284] avg=8192.0000 max=17066 max_rank=3 min=1359 min_rank=4 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[9087, 21876, 1721, 7065, 7819, 3960, 7691, 6317] avg=8192.0000 max=21876 max_rank=1 min=1721 min_rank=2 imbalance_ratio=2.670410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[11173, 1843, 8958, 17066, 1359, 12417, 8436, 4284] avg=8192.0000 max=17066 max_rank=3 min=1359 min_rank=4 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] avg=8192.0000 max=9213 max_rank=0 min=5048 min_rank=1 imbalance_ratio=1.124634 max_load_reduction=7853 max_load_reduction_pct=0.460155 moved_to_replicas=24564 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[11173, 1843, 8958, 17066, 1359, 12417, 8436, 4284] avg=8192.0000 max=17066 max_rank=3 min=1359 min_rank=4 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] avg=8192.0000 max=9213 max_rank=0 min=5048 min_rank=1 imbalance_ratio=1.124634 max_load_reduction=7853 max_load_reduction_pct=0.460155 moved_to_replicas=24564 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[11173, 1843, 8958, 17066, 1359, 12417, 8436, 4284] avg=8192.0000 max=17066 max_rank=3 min=1359 min_rank=4 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[9087, 21876, 1721, 7065, 7819, 3960, 7691, 6317] avg=8192.0000 max=21876 max_rank=1 min=1721 min_rank=2 imbalance_ratio=2.670410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] avg=8192.0000 max=8274 max_rank=0 min=7843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13602 max_load_reduction_pct=0.621777 moved_to_replicas=33807 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[11173, 1843, 8958, 17066, 1359, 12417, 8436, 4284] avg=8192.0000 max=17066 max_rank=3 min=1359 min_rank=4 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] avg=8192.0000 max=9213 max_rank=0 min=5048 min_rank=1 imbalance_ratio=1.124634 max_load_reduction=7853 max_load_reduction_pct=0.460155 moved_to_replicas=24564 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] avg=8192.0000 max=9213 max_rank=0 min=5048 min_rank=1 imbalance_ratio=1.124634 max_load_reduction=7853 max_load_reduction_pct=0.460155 moved_to_replicas=24564 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] avg=8192.0000 max=9213 max_rank=0 min=5048 min_rank=1 imbalance_ratio=1.124634 max_load_reduction=7853 max_load_reduction_pct=0.460155 moved_to_replicas=24564 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[9087, 21876, 1721, 7065, 7819, 3960, 7691, 6317] avg=8192.0000 max=21876 max_rank=1 min=1721 min_rank=2 imbalance_ratio=2.670410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] avg=8192.0000 max=8274 max_rank=0 min=7843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13602 max_load_reduction_pct=0.621777 moved_to_replicas=33807 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] avg=8192.0000 max=9213 max_rank=0 min=5048 min_rank=1 imbalance_ratio=1.124634 max_load_reduction=7853 max_load_reduction_pct=0.460155 moved_to_replicas=24564 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[9087, 21876, 1721, 7065, 7819, 3960, 7691, 6317] avg=8192.0000 max=21876 max_rank=1 min=1721 min_rank=2 imbalance_ratio=2.670410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[15475, 6841, 6249, 8271, 6238, 6916, 6203, 9343] avg=8192.0000 max=15475 max_rank=0 min=6203 min_rank=6 imbalance_ratio=1.889038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[9087, 21876, 1721, 7065, 7819, 3960, 7691, 6317] avg=8192.0000 max=21876 max_rank=1 min=1721 min_rank=2 imbalance_ratio=2.670410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] avg=8192.0000 max=8274 max_rank=0 min=7843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13602 max_load_reduction_pct=0.621777 moved_to_replicas=33807 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[9087, 21876, 1721, 7065, 7819, 3960, 7691, 6317] avg=8192.0000 max=21876 max_rank=1 min=1721 min_rank=2 imbalance_ratio=2.670410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[9087, 21876, 1721, 7065, 7819, 3960, 7691, 6317] avg=8192.0000 max=21876 max_rank=1 min=1721 min_rank=2 imbalance_ratio=2.670410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] avg=8192.0000 max=8274 max_rank=0 min=7843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13602 max_load_reduction_pct=0.621777 moved_to_replicas=33807 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[15475, 6841, 6249, 8271, 6238, 6916, 6203, 9343] avg=8192.0000 max=15475 max_rank=0 min=6203 min_rank=6 imbalance_ratio=1.889038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] avg=8192.0000 max=8274 max_rank=2 min=7911 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7201 max_load_reduction_pct=0.465331 moved_to_replicas=14018 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[9087, 21876, 1721, 7065, 7819, 3960, 7691, 6317] avg=8192.0000 max=21876 max_rank=1 min=1721 min_rank=2 imbalance_ratio=2.670410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] avg=8192.0000 max=8274 max_rank=0 min=7843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13602 max_load_reduction_pct=0.621777 moved_to_replicas=33807 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] avg=8192.0000 max=8274 max_rank=0 min=7843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13602 max_load_reduction_pct=0.621777 moved_to_replicas=33807 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[15475, 6841, 6249, 8271, 6238, 6916, 6203, 9343] avg=8192.0000 max=15475 max_rank=0 min=6203 min_rank=6 imbalance_ratio=1.889038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] avg=8192.0000 max=8274 max_rank=0 min=7843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13602 max_load_reduction_pct=0.621777 moved_to_replicas=33807 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] avg=8192.0000 max=8274 max_rank=2 min=7911 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7201 max_load_reduction_pct=0.465331 moved_to_replicas=14018 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] avg=8192.0000 max=8274 max_rank=0 min=7843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13602 max_load_reduction_pct=0.621777 moved_to_replicas=33807 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[15475, 6841, 6249, 8271, 6238, 6916, 6203, 9343] avg=8192.0000 max=15475 max_rank=0 min=6203 min_rank=6 imbalance_ratio=1.889038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[5092, 6103, 5082, 7798, 17478, 9628, 6699, 7656] avg=8192.0000 max=17478 max_rank=4 min=5082 min_rank=2 imbalance_ratio=2.133545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[15475, 6841, 6249, 8271, 6238, 6916, 6203, 9343] avg=8192.0000 max=15475 max_rank=0 min=6203 min_rank=6 imbalance_ratio=1.889038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] avg=8192.0000 max=8274 max_rank=2 min=7911 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7201 max_load_reduction_pct=0.465331 moved_to_replicas=14018 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[15475, 6841, 6249, 8271, 6238, 6916, 6203, 9343] avg=8192.0000 max=15475 max_rank=0 min=6203 min_rank=6 imbalance_ratio=1.889038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[15475, 6841, 6249, 8271, 6238, 6916, 6203, 9343] avg=8192.0000 max=15475 max_rank=0 min=6203 min_rank=6 imbalance_ratio=1.889038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] avg=8192.0000 max=8274 max_rank=2 min=7911 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7201 max_load_reduction_pct=0.465331 moved_to_replicas=14018 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] avg=8192.0000 max=8450 max_rank=2 min=7656 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=9028 max_load_reduction_pct=0.516535 moved_to_replicas=22428 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[5092, 6103, 5082, 7798, 17478, 9628, 6699, 7656] avg=8192.0000 max=17478 max_rank=4 min=5082 min_rank=2 imbalance_ratio=2.133545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] avg=8192.0000 max=8274 max_rank=2 min=7911 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7201 max_load_reduction_pct=0.465331 moved_to_replicas=14018 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[15475, 6841, 6249, 8271, 6238, 6916, 6203, 9343] avg=8192.0000 max=15475 max_rank=0 min=6203 min_rank=6 imbalance_ratio=1.889038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] avg=8192.0000 max=8274 max_rank=2 min=7911 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7201 max_load_reduction_pct=0.465331 moved_to_replicas=14018 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[5092, 6103, 5082, 7798, 17478, 9628, 6699, 7656] avg=8192.0000 max=17478 max_rank=4 min=5082 min_rank=2 imbalance_ratio=2.133545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] avg=8192.0000 max=8274 max_rank=2 min=7911 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7201 max_load_reduction_pct=0.465331 moved_to_replicas=14018 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] avg=8192.0000 max=8450 max_rank=2 min=7656 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=9028 max_load_reduction_pct=0.516535 moved_to_replicas=22428 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] avg=8192.0000 max=8274 max_rank=2 min=7911 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7201 max_load_reduction_pct=0.465331 moved_to_replicas=14018 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[5092, 6103, 5082, 7798, 17478, 9628, 6699, 7656] avg=8192.0000 max=17478 max_rank=4 min=5082 min_rank=2 imbalance_ratio=2.133545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[17476, 826, 1489, 1964, 1703, 7237, 16410, 18431] avg=8192.0000 max=18431 max_rank=7 min=826 min_rank=1 imbalance_ratio=2.249878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[5092, 6103, 5082, 7798, 17478, 9628, 6699, 7656] avg=8192.0000 max=17478 max_rank=4 min=5082 min_rank=2 imbalance_ratio=2.133545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] avg=8192.0000 max=8450 max_rank=2 min=7656 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=9028 max_load_reduction_pct=0.516535 moved_to_replicas=22428 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[5092, 6103, 5082, 7798, 17478, 9628, 6699, 7656] avg=8192.0000 max=17478 max_rank=4 min=5082 min_rank=2 imbalance_ratio=2.133545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[5092, 6103, 5082, 7798, 17478, 9628, 6699, 7656] avg=8192.0000 max=17478 max_rank=4 min=5082 min_rank=2 imbalance_ratio=2.133545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] avg=8192.0000 max=8450 max_rank=2 min=7656 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=9028 max_load_reduction_pct=0.516535 moved_to_replicas=22428 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[17476, 826, 1489, 1964, 1703, 7237, 16410, 18431] avg=8192.0000 max=18431 max_rank=7 min=826 min_rank=1 imbalance_ratio=2.249878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] avg=8192.0000 max=9630 max_rank=0 min=1964 min_rank=3 imbalance_ratio=1.175537 max_load_reduction=8801 max_load_reduction_pct=0.477511 moved_to_replicas=33952 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] avg=8192.0000 max=8450 max_rank=2 min=7656 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=9028 max_load_reduction_pct=0.516535 moved_to_replicas=22428 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[5092, 6103, 5082, 7798, 17478, 9628, 6699, 7656] avg=8192.0000 max=17478 max_rank=4 min=5082 min_rank=2 imbalance_ratio=2.133545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] avg=8192.0000 max=8450 max_rank=2 min=7656 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=9028 max_load_reduction_pct=0.516535 moved_to_replicas=22428 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] avg=8192.0000 max=8450 max_rank=2 min=7656 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=9028 max_load_reduction_pct=0.516535 moved_to_replicas=22428 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] avg=8192.0000 max=9630 max_rank=0 min=1964 min_rank=3 imbalance_ratio=1.175537 max_load_reduction=8801 max_load_reduction_pct=0.477511 moved_to_replicas=33952 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[17476, 826, 1489, 1964, 1703, 7237, 16410, 18431] avg=8192.0000 max=18431 max_rank=7 min=826 min_rank=1 imbalance_ratio=2.249878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] avg=8192.0000 max=8450 max_rank=2 min=7656 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=9028 max_load_reduction_pct=0.516535 moved_to_replicas=22428 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[17476, 826, 1489, 1964, 1703, 7237, 16410, 18431] avg=8192.0000 max=18431 max_rank=7 min=826 min_rank=1 imbalance_ratio=2.249878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[14663, 5495, 14370, 4817, 10121, 6131, 2345, 7594] avg=8192.0000 max=14663 max_rank=0 min=2345 min_rank=6 imbalance_ratio=1.789917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[17476, 826, 1489, 1964, 1703, 7237, 16410, 18431] avg=8192.0000 max=18431 max_rank=7 min=826 min_rank=1 imbalance_ratio=2.249878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] avg=8192.0000 max=9630 max_rank=0 min=1964 min_rank=3 imbalance_ratio=1.175537 max_load_reduction=8801 max_load_reduction_pct=0.477511 moved_to_replicas=33952 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[17476, 826, 1489, 1964, 1703, 7237, 16410, 18431] avg=8192.0000 max=18431 max_rank=7 min=826 min_rank=1 imbalance_ratio=2.249878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[17476, 826, 1489, 1964, 1703, 7237, 16410, 18431] avg=8192.0000 max=18431 max_rank=7 min=826 min_rank=1 imbalance_ratio=2.249878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] avg=8192.0000 max=9630 max_rank=0 min=1964 min_rank=3 imbalance_ratio=1.175537 max_load_reduction=8801 max_load_reduction_pct=0.477511 moved_to_replicas=33952 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[14663, 5495, 14370, 4817, 10121, 6131, 2345, 7594] avg=8192.0000 max=14663 max_rank=0 min=2345 min_rank=6 imbalance_ratio=1.789917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] avg=8192.0000 max=8569 max_rank=3 min=7546 min_rank=1 imbalance_ratio=1.046021 max_load_reduction=6094 max_load_reduction_pct=0.415604 moved_to_replicas=19831 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] avg=8192.0000 max=9630 max_rank=0 min=1964 min_rank=3 imbalance_ratio=1.175537 max_load_reduction=8801 max_load_reduction_pct=0.477511 moved_to_replicas=33952 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[17476, 826, 1489, 1964, 1703, 7237, 16410, 18431] avg=8192.0000 max=18431 max_rank=7 min=826 min_rank=1 imbalance_ratio=2.249878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] avg=8192.0000 max=9630 max_rank=0 min=1964 min_rank=3 imbalance_ratio=1.175537 max_load_reduction=8801 max_load_reduction_pct=0.477511 moved_to_replicas=33952 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] avg=8192.0000 max=9630 max_rank=0 min=1964 min_rank=3 imbalance_ratio=1.175537 max_load_reduction=8801 max_load_reduction_pct=0.477511 moved_to_replicas=33952 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] avg=8192.0000 max=8569 max_rank=3 min=7546 min_rank=1 imbalance_ratio=1.046021 max_load_reduction=6094 max_load_reduction_pct=0.415604 moved_to_replicas=19831 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[14663, 5495, 14370, 4817, 10121, 6131, 2345, 7594] avg=8192.0000 max=14663 max_rank=0 min=2345 min_rank=6 imbalance_ratio=1.789917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] avg=8192.0000 max=9630 max_rank=0 min=1964 min_rank=3 imbalance_ratio=1.175537 max_load_reduction=8801 max_load_reduction_pct=0.477511 moved_to_replicas=33952 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[14663, 5495, 14370, 4817, 10121, 6131, 2345, 7594] avg=8192.0000 max=14663 max_rank=0 min=2345 min_rank=6 imbalance_ratio=1.789917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 20789, 2987, 1890, 12413, 10818] avg=8192.0000 max=20789 max_rank=3 min=1890 min_rank=5 imbalance_ratio=2.537720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[14663, 5495, 14370, 4817, 10121, 6131, 2345, 7594] avg=8192.0000 max=14663 max_rank=0 min=2345 min_rank=6 imbalance_ratio=1.789917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] avg=8192.0000 max=8569 max_rank=3 min=7546 min_rank=1 imbalance_ratio=1.046021 max_load_reduction=6094 max_load_reduction_pct=0.415604 moved_to_replicas=19831 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[14663, 5495, 14370, 4817, 10121, 6131, 2345, 7594] avg=8192.0000 max=14663 max_rank=0 min=2345 min_rank=6 imbalance_ratio=1.789917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[14663, 5495, 14370, 4817, 10121, 6131, 2345, 7594] avg=8192.0000 max=14663 max_rank=0 min=2345 min_rank=6 imbalance_ratio=1.789917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 20789, 2987, 1890, 12413, 10818] avg=8192.0000 max=20789 max_rank=3 min=1890 min_rank=5 imbalance_ratio=2.537720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] avg=8192.0000 max=8569 max_rank=3 min=7546 min_rank=1 imbalance_ratio=1.046021 max_load_reduction=6094 max_load_reduction_pct=0.415604 moved_to_replicas=19831 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[14663, 5495, 14370, 4817, 10121, 6131, 2345, 7594] avg=8192.0000 max=14663 max_rank=0 min=2345 min_rank=6 imbalance_ratio=1.789917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] avg=8192.0000 max=12413 max_rank=3 min=2820 min_rank=1 imbalance_ratio=1.515259 max_load_reduction=8376 max_load_reduction_pct=0.402905 moved_to_replicas=20961 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] avg=8192.0000 max=8569 max_rank=3 min=7546 min_rank=1 imbalance_ratio=1.046021 max_load_reduction=6094 max_load_reduction_pct=0.415604 moved_to_replicas=19831 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] avg=8192.0000 max=8569 max_rank=3 min=7546 min_rank=1 imbalance_ratio=1.046021 max_load_reduction=6094 max_load_reduction_pct=0.415604 moved_to_replicas=19831 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] avg=8192.0000 max=12413 max_rank=3 min=2820 min_rank=1 imbalance_ratio=1.515259 max_load_reduction=8376 max_load_reduction_pct=0.402905 moved_to_replicas=20961 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] avg=8192.0000 max=8569 max_rank=3 min=7546 min_rank=1 imbalance_ratio=1.046021 max_load_reduction=6094 max_load_reduction_pct=0.415604 moved_to_replicas=19831 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 20789, 2987, 1890, 12413, 10818] avg=8192.0000 max=20789 max_rank=3 min=1890 min_rank=5 imbalance_ratio=2.537720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] avg=8192.0000 max=8569 max_rank=3 min=7546 min_rank=1 imbalance_ratio=1.046021 max_load_reduction=6094 max_load_reduction_pct=0.415604 moved_to_replicas=19831 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 20789, 2987, 1890, 12413, 10818] avg=8192.0000 max=20789 max_rank=3 min=1890 min_rank=5 imbalance_ratio=2.537720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[9023, 17727, 1614, 6296, 2529, 3654, 16999, 7694] avg=8192.0000 max=17727 max_rank=1 min=1614 min_rank=2 imbalance_ratio=2.163940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 20789, 2987, 1890, 12413, 10818] avg=8192.0000 max=20789 max_rank=3 min=1890 min_rank=5 imbalance_ratio=2.537720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] avg=8192.0000 max=12413 max_rank=3 min=2820 min_rank=1 imbalance_ratio=1.515259 max_load_reduction=8376 max_load_reduction_pct=0.402905 moved_to_replicas=20961 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 20789, 2987, 1890, 12413, 10818] avg=8192.0000 max=20789 max_rank=3 min=1890 min_rank=5 imbalance_ratio=2.537720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[9023, 17727, 1614, 6296, 2529, 3654, 16999, 7694] avg=8192.0000 max=17727 max_rank=1 min=1614 min_rank=2 imbalance_ratio=2.163940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] avg=8192.0000 max=12413 max_rank=3 min=2820 min_rank=1 imbalance_ratio=1.515259 max_load_reduction=8376 max_load_reduction_pct=0.402905 moved_to_replicas=20961 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 20789, 2987, 1890, 12413, 10818] avg=8192.0000 max=20789 max_rank=3 min=1890 min_rank=5 imbalance_ratio=2.537720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 20789, 2987, 1890, 12413, 10818] avg=8192.0000 max=20789 max_rank=3 min=1890 min_rank=5 imbalance_ratio=2.537720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[9023, 9763, 9578, 6296, 9764, 3654, 9764, 7694] avg=8192.0000 max=9764 max_rank=4 min=3654 min_rank=5 imbalance_ratio=1.191895 max_load_reduction=7963 max_load_reduction_pct=0.449202 moved_to_replicas=29697 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] avg=8192.0000 max=12413 max_rank=3 min=2820 min_rank=1 imbalance_ratio=1.515259 max_load_reduction=8376 max_load_reduction_pct=0.402905 moved_to_replicas=20961 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] avg=8192.0000 max=12413 max_rank=3 min=2820 min_rank=1 imbalance_ratio=1.515259 max_load_reduction=8376 max_load_reduction_pct=0.402905 moved_to_replicas=20961 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[9023, 9763, 9578, 6296, 9764, 3654, 9764, 7694] avg=8192.0000 max=9764 max_rank=4 min=3654 min_rank=5 imbalance_ratio=1.191895 max_load_reduction=7963 max_load_reduction_pct=0.449202 moved_to_replicas=29697 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[9023, 17727, 1614, 6296, 2529, 3654, 16999, 7694] avg=8192.0000 max=17727 max_rank=1 min=1614 min_rank=2 imbalance_ratio=2.163940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] avg=8192.0000 max=12413 max_rank=3 min=2820 min_rank=1 imbalance_ratio=1.515259 max_load_reduction=8376 max_load_reduction_pct=0.402905 moved_to_replicas=20961 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] avg=8192.0000 max=12413 max_rank=3 min=2820 min_rank=1 imbalance_ratio=1.515259 max_load_reduction=8376 max_load_reduction_pct=0.402905 moved_to_replicas=20961 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[9023, 17727, 1614, 6296, 2529, 3654, 16999, 7694] avg=8192.0000 max=17727 max_rank=1 min=1614 min_rank=2 imbalance_ratio=2.163940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[22205, 12801, 5525, 6772, 350, 2574, 1419, 13890] avg=8192.0000 max=22205 max_rank=0 min=350 min_rank=4 imbalance_ratio=2.710571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[9023, 17727, 1614, 6296, 2529, 3654, 16999, 7694] avg=8192.0000 max=17727 max_rank=1 min=1614 min_rank=2 imbalance_ratio=2.163940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[9023, 17727, 1614, 6296, 2529, 3654, 16999, 7694] avg=8192.0000 max=17727 max_rank=1 min=1614 min_rank=2 imbalance_ratio=2.163940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[9023, 9763, 9578, 6296, 9764, 3654, 9764, 7694] avg=8192.0000 max=9764 max_rank=4 min=3654 min_rank=5 imbalance_ratio=1.191895 max_load_reduction=7963 max_load_reduction_pct=0.449202 moved_to_replicas=29697 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[22205, 12801, 5525, 6772, 350, 2574, 1419, 13890] avg=8192.0000 max=22205 max_rank=0 min=350 min_rank=4 imbalance_ratio=2.710571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[9023, 9763, 9578, 6296, 9764, 3654, 9764, 7694] avg=8192.0000 max=9764 max_rank=4 min=3654 min_rank=5 imbalance_ratio=1.191895 max_load_reduction=7963 max_load_reduction_pct=0.449202 moved_to_replicas=29697 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[9023, 17727, 1614, 6296, 2529, 3654, 16999, 7694] avg=8192.0000 max=17727 max_rank=1 min=1614 min_rank=2 imbalance_ratio=2.163940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[9023, 17727, 1614, 6296, 2529, 3654, 16999, 7694] avg=8192.0000 max=17727 max_rank=1 min=1614 min_rank=2 imbalance_ratio=2.163940 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[8497, 8497, 8497, 8104, 8497, 7967, 6980, 8497] avg=8192.0000 max=8497 max_rank=0 min=6980 min_rank=6 imbalance_ratio=1.037231 max_load_reduction=13708 max_load_reduction_pct=0.617338 moved_to_replicas=43041 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[9023, 9763, 9578, 6296, 9764, 3654, 9764, 7694] avg=8192.0000 max=9764 max_rank=4 min=3654 min_rank=5 imbalance_ratio=1.191895 max_load_reduction=7963 max_load_reduction_pct=0.449202 moved_to_replicas=29697 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[9023, 9763, 9578, 6296, 9764, 3654, 9764, 7694] avg=8192.0000 max=9764 max_rank=4 min=3654 min_rank=5 imbalance_ratio=1.191895 max_load_reduction=7963 max_load_reduction_pct=0.449202 moved_to_replicas=29697 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[8497, 8497, 8497, 8104, 8497, 7967, 6980, 8497] avg=8192.0000 max=8497 max_rank=0 min=6980 min_rank=6 imbalance_ratio=1.037231 max_load_reduction=13708 max_load_reduction_pct=0.617338 moved_to_replicas=43041 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[22205, 12801, 5525, 6772, 350, 2574, 1419, 13890] avg=8192.0000 max=22205 max_rank=0 min=350 min_rank=4 imbalance_ratio=2.710571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[9023, 9763, 9578, 6296, 9764, 3654, 9764, 7694] avg=8192.0000 max=9764 max_rank=4 min=3654 min_rank=5 imbalance_ratio=1.191895 max_load_reduction=7963 max_load_reduction_pct=0.449202 moved_to_replicas=29697 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[9023, 9763, 9578, 6296, 9764, 3654, 9764, 7694] avg=8192.0000 max=9764 max_rank=4 min=3654 min_rank=5 imbalance_ratio=1.191895 max_load_reduction=7963 max_load_reduction_pct=0.449202 moved_to_replicas=29697 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[22205, 12801, 5525, 6772, 350, 2574, 1419, 13890] avg=8192.0000 max=22205 max_rank=0 min=350 min_rank=4 imbalance_ratio=2.710571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[14879, 2453, 4806, 17583, 3565, 11579, 9285, 1386] avg=8192.0000 max=17583 max_rank=3 min=1386 min_rank=7 imbalance_ratio=2.146362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[22205, 12801, 5525, 6772, 350, 2574, 1419, 13890] avg=8192.0000 max=22205 max_rank=0 min=350 min_rank=4 imbalance_ratio=2.710571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[22205, 12801, 5525, 6772, 350, 2574, 1419, 13890] avg=8192.0000 max=22205 max_rank=0 min=350 min_rank=4 imbalance_ratio=2.710571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[8497, 8497, 8497, 8104, 8497, 7967, 6980, 8497] avg=8192.0000 max=8497 max_rank=0 min=6980 min_rank=6 imbalance_ratio=1.037231 max_load_reduction=13708 max_load_reduction_pct=0.617338 moved_to_replicas=43041 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[14879, 2453, 4806, 17583, 3565, 11579, 9285, 1386] avg=8192.0000 max=17583 max_rank=3 min=1386 min_rank=7 imbalance_ratio=2.146362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[8497, 8497, 8497, 8104, 8497, 7967, 6980, 8497] avg=8192.0000 max=8497 max_rank=0 min=6980 min_rank=6 imbalance_ratio=1.037231 max_load_reduction=13708 max_load_reduction_pct=0.617338 moved_to_replicas=43041 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[22205, 12801, 5525, 6772, 350, 2574, 1419, 13890] avg=8192.0000 max=22205 max_rank=0 min=350 min_rank=4 imbalance_ratio=2.710571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[22205, 12801, 5525, 6772, 350, 2574, 1419, 13890] avg=8192.0000 max=22205 max_rank=0 min=350 min_rank=4 imbalance_ratio=2.710571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[9486, 7846, 4806, 9484, 5659, 9485, 9285, 9485] avg=8192.0000 max=9486 max_rank=0 min=4806 min_rank=2 imbalance_ratio=1.157959 max_load_reduction=8097 max_load_reduction_pct=0.460502 moved_to_replicas=25605 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[8497, 8497, 8497, 8104, 8497, 7967, 6980, 8497] avg=8192.0000 max=8497 max_rank=0 min=6980 min_rank=6 imbalance_ratio=1.037231 max_load_reduction=13708 max_load_reduction_pct=0.617338 moved_to_replicas=43041 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[8497, 8497, 8497, 8104, 8497, 7967, 6980, 8497] avg=8192.0000 max=8497 max_rank=0 min=6980 min_rank=6 imbalance_ratio=1.037231 max_load_reduction=13708 max_load_reduction_pct=0.617338 moved_to_replicas=43041 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[9486, 7846, 4806, 9484, 5659, 9485, 9285, 9485] avg=8192.0000 max=9486 max_rank=0 min=4806 min_rank=2 imbalance_ratio=1.157959 max_load_reduction=8097 max_load_reduction_pct=0.460502 moved_to_replicas=25605 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[8497, 8497, 8497, 8104, 8497, 7967, 6980, 8497] avg=8192.0000 max=8497 max_rank=0 min=6980 min_rank=6 imbalance_ratio=1.037231 max_load_reduction=13708 max_load_reduction_pct=0.617338 moved_to_replicas=43041 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[14879, 2453, 4806, 17583, 3565, 11579, 9285, 1386] avg=8192.0000 max=17583 max_rank=3 min=1386 min_rank=7 imbalance_ratio=2.146362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[8497, 8497, 8497, 8104, 8497, 7967, 6980, 8497] avg=8192.0000 max=8497 max_rank=0 min=6980 min_rank=6 imbalance_ratio=1.037231 max_load_reduction=13708 max_load_reduction_pct=0.617338 moved_to_replicas=43041 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[14879, 2453, 4806, 17583, 3565, 11579, 9285, 1386] avg=8192.0000 max=17583 max_rank=3 min=1386 min_rank=7 imbalance_ratio=2.146362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[9230, 12344, 4882, 18764, 5185, 2783, 1981, 10367] avg=8192.0000 max=18764 max_rank=3 min=1981 min_rank=6 imbalance_ratio=2.290527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[14879, 2453, 4806, 17583, 3565, 11579, 9285, 1386] avg=8192.0000 max=17583 max_rank=3 min=1386 min_rank=7 imbalance_ratio=2.146362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[14879, 2453, 4806, 17583, 3565, 11579, 9285, 1386] avg=8192.0000 max=17583 max_rank=3 min=1386 min_rank=7 imbalance_ratio=2.146362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[9486, 7846, 4806, 9484, 5659, 9485, 9285, 9485] avg=8192.0000 max=9486 max_rank=0 min=4806 min_rank=2 imbalance_ratio=1.157959 max_load_reduction=8097 max_load_reduction_pct=0.460502 moved_to_replicas=25605 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[9486, 7846, 4806, 9484, 5659, 9485, 9285, 9485] avg=8192.0000 max=9486 max_rank=0 min=4806 min_rank=2 imbalance_ratio=1.157959 max_load_reduction=8097 max_load_reduction_pct=0.460502 moved_to_replicas=25605 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[14879, 2453, 4806, 17583, 3565, 11579, 9285, 1386] avg=8192.0000 max=17583 max_rank=3 min=1386 min_rank=7 imbalance_ratio=2.146362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[9230, 10373, 4882, 10372, 5185, 4754, 10373, 10367] avg=8192.0000 max=10373 max_rank=1 min=4754 min_rank=5 imbalance_ratio=1.266235 max_load_reduction=8391 max_load_reduction_pct=0.447186 moved_to_replicas=25263 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[14879, 2453, 4806, 17583, 3565, 11579, 9285, 1386] avg=8192.0000 max=17583 max_rank=3 min=1386 min_rank=7 imbalance_ratio=2.146362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[9486, 7846, 4806, 9484, 5659, 9485, 9285, 9485] avg=8192.0000 max=9486 max_rank=0 min=4806 min_rank=2 imbalance_ratio=1.157959 max_load_reduction=8097 max_load_reduction_pct=0.460502 moved_to_replicas=25605 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[9230, 12344, 4882, 18764, 5185, 2783, 1981, 10367] avg=8192.0000 max=18764 max_rank=3 min=1981 min_rank=6 imbalance_ratio=2.290527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[9486, 7846, 4806, 9484, 5659, 9485, 9285, 9485] avg=8192.0000 max=9486 max_rank=0 min=4806 min_rank=2 imbalance_ratio=1.157959 max_load_reduction=8097 max_load_reduction_pct=0.460502 moved_to_replicas=25605 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[9486, 7846, 4806, 9484, 5659, 9485, 9285, 9485] avg=8192.0000 max=9486 max_rank=0 min=4806 min_rank=2 imbalance_ratio=1.157959 max_load_reduction=8097 max_load_reduction_pct=0.460502 moved_to_replicas=25605 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[9230, 12344, 4882, 18764, 5185, 2783, 1981, 10367] avg=8192.0000 max=18764 max_rank=3 min=1981 min_rank=6 imbalance_ratio=2.290527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[9486, 7846, 4806, 9484, 5659, 9485, 9285, 9485] avg=8192.0000 max=9486 max_rank=0 min=4806 min_rank=2 imbalance_ratio=1.157959 max_load_reduction=8097 max_load_reduction_pct=0.460502 moved_to_replicas=25605 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[9230, 10373, 4882, 10372, 5185, 4754, 10373, 10367] avg=8192.0000 max=10373 max_rank=1 min=4754 min_rank=5 imbalance_ratio=1.266235 max_load_reduction=8391 max_load_reduction_pct=0.447186 moved_to_replicas=25263 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[9230, 12344, 4882, 18764, 5185, 2783, 1981, 10367] avg=8192.0000 max=18764 max_rank=3 min=1981 min_rank=6 imbalance_ratio=2.290527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[2470, 2604, 11696, 16412, 12535, 2973, 5189, 11657] avg=8192.0000 max=16412 max_rank=3 min=2470 min_rank=0 imbalance_ratio=2.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[9230, 12344, 4882, 18764, 5185, 2783, 1981, 10367] avg=8192.0000 max=18764 max_rank=3 min=1981 min_rank=6 imbalance_ratio=2.290527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[9230, 12344, 4882, 18764, 5185, 2783, 1981, 10367] avg=8192.0000 max=18764 max_rank=3 min=1981 min_rank=6 imbalance_ratio=2.290527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[9230, 10373, 4882, 10372, 5185, 4754, 10373, 10367] avg=8192.0000 max=10373 max_rank=1 min=4754 min_rank=5 imbalance_ratio=1.266235 max_load_reduction=8391 max_load_reduction_pct=0.447186 moved_to_replicas=25263 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[9230, 10373, 4882, 10372, 5185, 4754, 10373, 10367] avg=8192.0000 max=10373 max_rank=1 min=4754 min_rank=5 imbalance_ratio=1.266235 max_load_reduction=8391 max_load_reduction_pct=0.447186 moved_to_replicas=25263 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[9230, 12344, 4882, 18764, 5185, 2783, 1981, 10367] avg=8192.0000 max=18764 max_rank=3 min=1981 min_rank=6 imbalance_ratio=2.290527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[9442, 5698, 9441, 9440, 9441, 5228, 7405, 9441] avg=8192.0000 max=9442 max_rank=0 min=5228 min_rank=5 imbalance_ratio=1.152588 max_load_reduction=6970 max_load_reduction_pct=0.424689 moved_to_replicas=20490 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[9230, 10373, 4882, 10372, 5185, 4754, 10373, 10367] avg=8192.0000 max=10373 max_rank=1 min=4754 min_rank=5 imbalance_ratio=1.266235 max_load_reduction=8391 max_load_reduction_pct=0.447186 moved_to_replicas=25263 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[9230, 12344, 4882, 18764, 5185, 2783, 1981, 10367] avg=8192.0000 max=18764 max_rank=3 min=1981 min_rank=6 imbalance_ratio=2.290527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[2470, 2604, 11696, 16412, 12535, 2973, 5189, 11657] avg=8192.0000 max=16412 max_rank=3 min=2470 min_rank=0 imbalance_ratio=2.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[9230, 10373, 4882, 10372, 5185, 4754, 10373, 10367] avg=8192.0000 max=10373 max_rank=1 min=4754 min_rank=5 imbalance_ratio=1.266235 max_load_reduction=8391 max_load_reduction_pct=0.447186 moved_to_replicas=25263 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[9230, 10373, 4882, 10372, 5185, 4754, 10373, 10367] avg=8192.0000 max=10373 max_rank=1 min=4754 min_rank=5 imbalance_ratio=1.266235 max_load_reduction=8391 max_load_reduction_pct=0.447186 moved_to_replicas=25263 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[2470, 2604, 11696, 16412, 12535, 2973, 5189, 11657] avg=8192.0000 max=16412 max_rank=3 min=2470 min_rank=0 imbalance_ratio=2.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[9230, 10373, 4882, 10372, 5185, 4754, 10373, 10367] avg=8192.0000 max=10373 max_rank=1 min=4754 min_rank=5 imbalance_ratio=1.266235 max_load_reduction=8391 max_load_reduction_pct=0.447186 moved_to_replicas=25263 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[9442, 5698, 9441, 9440, 9441, 5228, 7405, 9441] avg=8192.0000 max=9442 max_rank=0 min=5228 min_rank=5 imbalance_ratio=1.152588 max_load_reduction=6970 max_load_reduction_pct=0.424689 moved_to_replicas=20490 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[2470, 2604, 11696, 16412, 12535, 2973, 5189, 11657] avg=8192.0000 max=16412 max_rank=3 min=2470 min_rank=0 imbalance_ratio=2.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[1241, 7294, 7362, 11658, 3425, 17394, 5384, 11778] avg=8192.0000 max=17394 max_rank=5 min=1241 min_rank=0 imbalance_ratio=2.123291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[2470, 2604, 11696, 16412, 12535, 2973, 5189, 11657] avg=8192.0000 max=16412 max_rank=3 min=2470 min_rank=0 imbalance_ratio=2.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[2470, 2604, 11696, 16412, 12535, 2973, 5189, 11657] avg=8192.0000 max=16412 max_rank=3 min=2470 min_rank=0 imbalance_ratio=2.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[9442, 5698, 9441, 9440, 9441, 5228, 7405, 9441] avg=8192.0000 max=9442 max_rank=0 min=5228 min_rank=5 imbalance_ratio=1.152588 max_load_reduction=6970 max_load_reduction_pct=0.424689 moved_to_replicas=20490 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[9442, 5698, 9441, 9440, 9441, 5228, 7405, 9441] avg=8192.0000 max=9442 max_rank=0 min=5228 min_rank=5 imbalance_ratio=1.152588 max_load_reduction=6970 max_load_reduction_pct=0.424689 moved_to_replicas=20490 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[9316, 7294, 7362, 9318, 5885, 9319, 7724, 9318] avg=8192.0000 max=9319 max_rank=5 min=5885 min_rank=4 imbalance_ratio=1.137573 max_load_reduction=8075 max_load_reduction_pct=0.464241 moved_to_replicas=24298 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[9442, 5698, 9441, 9440, 9441, 5228, 7405, 9441] avg=8192.0000 max=9442 max_rank=0 min=5228 min_rank=5 imbalance_ratio=1.152588 max_load_reduction=6970 max_load_reduction_pct=0.424689 moved_to_replicas=20490 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[2470, 2604, 11696, 16412, 12535, 2973, 5189, 11657] avg=8192.0000 max=16412 max_rank=3 min=2470 min_rank=0 imbalance_ratio=2.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[2470, 2604, 11696, 16412, 12535, 2973, 5189, 11657] avg=8192.0000 max=16412 max_rank=3 min=2470 min_rank=0 imbalance_ratio=2.003418 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[9442, 5698, 9441, 9440, 9441, 5228, 7405, 9441] avg=8192.0000 max=9442 max_rank=0 min=5228 min_rank=5 imbalance_ratio=1.152588 max_load_reduction=6970 max_load_reduction_pct=0.424689 moved_to_replicas=20490 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[1241, 7294, 7362, 11658, 3425, 17394, 5384, 11778] avg=8192.0000 max=17394 max_rank=5 min=1241 min_rank=0 imbalance_ratio=2.123291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[1241, 7294, 7362, 11658, 3425, 17394, 5384, 11778] avg=8192.0000 max=17394 max_rank=5 min=1241 min_rank=0 imbalance_ratio=2.123291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[9442, 5698, 9441, 9440, 9441, 5228, 7405, 9441] avg=8192.0000 max=9442 max_rank=0 min=5228 min_rank=5 imbalance_ratio=1.152588 max_load_reduction=6970 max_load_reduction_pct=0.424689 moved_to_replicas=20490 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[9442, 5698, 9441, 9440, 9441, 5228, 7405, 9441] avg=8192.0000 max=9442 max_rank=0 min=5228 min_rank=5 imbalance_ratio=1.152588 max_load_reduction=6970 max_load_reduction_pct=0.424689 moved_to_replicas=20490 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[9316, 7294, 7362, 9318, 5885, 9319, 7724, 9318] avg=8192.0000 max=9319 max_rank=5 min=5885 min_rank=4 imbalance_ratio=1.137573 max_load_reduction=8075 max_load_reduction_pct=0.464241 moved_to_replicas=24298 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[1241, 7294, 7362, 11658, 3425, 17394, 5384, 11778] avg=8192.0000 max=17394 max_rank=5 min=1241 min_rank=0 imbalance_ratio=2.123291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[5744, 15601, 6322, 945, 5277, 17770, 3023, 10854] avg=8192.0000 max=17770 max_rank=5 min=945 min_rank=3 imbalance_ratio=2.169189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[1241, 7294, 7362, 11658, 3425, 17394, 5384, 11778] avg=8192.0000 max=17394 max_rank=5 min=1241 min_rank=0 imbalance_ratio=2.123291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[1241, 7294, 7362, 11658, 3425, 17394, 5384, 11778] avg=8192.0000 max=17394 max_rank=5 min=1241 min_rank=0 imbalance_ratio=2.123291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[9316, 7294, 7362, 9318, 5885, 9319, 7724, 9318] avg=8192.0000 max=9319 max_rank=5 min=5885 min_rank=4 imbalance_ratio=1.137573 max_load_reduction=8075 max_load_reduction_pct=0.464241 moved_to_replicas=24298 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[9316, 7294, 7362, 9318, 5885, 9319, 7724, 9318] avg=8192.0000 max=9319 max_rank=5 min=5885 min_rank=4 imbalance_ratio=1.137573 max_load_reduction=8075 max_load_reduction_pct=0.464241 moved_to_replicas=24298 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[5744, 10854, 6322, 7859, 5277, 10856, 7770, 10854] avg=8192.0000 max=10856 max_rank=5 min=5277 min_rank=4 imbalance_ratio=1.325195 max_load_reduction=6914 max_load_reduction_pct=0.389083 moved_to_replicas=23075 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[1241, 7294, 7362, 11658, 3425, 17394, 5384, 11778] avg=8192.0000 max=17394 max_rank=5 min=1241 min_rank=0 imbalance_ratio=2.123291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[9316, 7294, 7362, 9318, 5885, 9319, 7724, 9318] avg=8192.0000 max=9319 max_rank=5 min=5885 min_rank=4 imbalance_ratio=1.137573 max_load_reduction=8075 max_load_reduction_pct=0.464241 moved_to_replicas=24298 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[1241, 7294, 7362, 11658, 3425, 17394, 5384, 11778] avg=8192.0000 max=17394 max_rank=5 min=1241 min_rank=0 imbalance_ratio=2.123291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[5744, 15601, 6322, 945, 5277, 17770, 3023, 10854] avg=8192.0000 max=17770 max_rank=5 min=945 min_rank=3 imbalance_ratio=2.169189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[9316, 7294, 7362, 9318, 5885, 9319, 7724, 9318] avg=8192.0000 max=9319 max_rank=5 min=5885 min_rank=4 imbalance_ratio=1.137573 max_load_reduction=8075 max_load_reduction_pct=0.464241 moved_to_replicas=24298 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[5744, 15601, 6322, 945, 5277, 17770, 3023, 10854] avg=8192.0000 max=17770 max_rank=5 min=945 min_rank=3 imbalance_ratio=2.169189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[9316, 7294, 7362, 9318, 5885, 9319, 7724, 9318] avg=8192.0000 max=9319 max_rank=5 min=5885 min_rank=4 imbalance_ratio=1.137573 max_load_reduction=8075 max_load_reduction_pct=0.464241 moved_to_replicas=24298 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[5744, 10854, 6322, 7859, 5277, 10856, 7770, 10854] avg=8192.0000 max=10856 max_rank=5 min=5277 min_rank=4 imbalance_ratio=1.325195 max_load_reduction=6914 max_load_reduction_pct=0.389083 moved_to_replicas=23075 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[9316, 7294, 7362, 9318, 5885, 9319, 7724, 9318] avg=8192.0000 max=9319 max_rank=5 min=5885 min_rank=4 imbalance_ratio=1.137573 max_load_reduction=8075 max_load_reduction_pct=0.464241 moved_to_replicas=24298 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[5744, 15601, 6322, 945, 5277, 17770, 3023, 10854] avg=8192.0000 max=17770 max_rank=5 min=945 min_rank=3 imbalance_ratio=2.169189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[5744, 15601, 6322, 945, 5277, 17770, 3023, 10854] avg=8192.0000 max=17770 max_rank=5 min=945 min_rank=3 imbalance_ratio=2.169189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[5744, 15601, 6322, 945, 5277, 17770, 3023, 10854] avg=8192.0000 max=17770 max_rank=5 min=945 min_rank=3 imbalance_ratio=2.169189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[5744, 10854, 6322, 7859, 5277, 10856, 7770, 10854] avg=8192.0000 max=10856 max_rank=5 min=5277 min_rank=4 imbalance_ratio=1.325195 max_load_reduction=6914 max_load_reduction_pct=0.389083 moved_to_replicas=23075 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[5744, 10854, 6322, 7859, 5277, 10856, 7770, 10854] avg=8192.0000 max=10856 max_rank=5 min=5277 min_rank=4 imbalance_ratio=1.325195 max_load_reduction=6914 max_load_reduction_pct=0.389083 moved_to_replicas=23075 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[5744, 10854, 6322, 7859, 5277, 10856, 7770, 10854] avg=8192.0000 max=10856 max_rank=5 min=5277 min_rank=4 imbalance_ratio=1.325195 max_load_reduction=6914 max_load_reduction_pct=0.389083 moved_to_replicas=23075 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[5744, 10854, 6322, 7859, 5277, 10856, 7770, 10854] avg=8192.0000 max=10856 max_rank=5 min=5277 min_rank=4 imbalance_ratio=1.325195 max_load_reduction=6914 max_load_reduction_pct=0.389083 moved_to_replicas=23075 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[5744, 15601, 6322, 945, 5277, 17770, 3023, 10854] avg=8192.0000 max=17770 max_rank=5 min=945 min_rank=3 imbalance_ratio=2.169189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[5744, 15601, 6322, 945, 5277, 17770, 3023, 10854] avg=8192.0000 max=17770 max_rank=5 min=945 min_rank=3 imbalance_ratio=2.169189 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[5744, 10854, 6322, 7859, 5277, 10856, 7770, 10854] avg=8192.0000 max=10856 max_rank=5 min=5277 min_rank=4 imbalance_ratio=1.325195 max_load_reduction=6914 max_load_reduction_pct=0.389083 moved_to_replicas=23075 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[5744, 10854, 6322, 7859, 5277, 10856, 7770, 10854] avg=8192.0000 max=10856 max_rank=5 min=5277 min_rank=4 imbalance_ratio=1.325195 max_load_reduction=6914 max_load_reduction_pct=0.389083 moved_to_replicas=23075 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=32 total_expected_recv=269769 total_dropped_tokens=5459 drop_rate_pct=2.023583 records_with_drop_pct=12.500000 peak_rank_overflow=2484 total_slack_slots=61056 peak_demand=12724 required_capacity=1.553223 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=32 total_expected_recv=271113 total_dropped_tokens=3268 drop_rate_pct=1.205401 records_with_drop_pct=18.750000 peak_rank_overflow=864 total_slack_slots=57856 peak_demand=11104 required_capacity=1.355469 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=32 total_expected_recv=247010 total_dropped_tokens=1599 drop_rate_pct=0.647342 records_with_drop_pct=9.375000 peak_rank_overflow=852 total_slack_slots=78976 peak_demand=11092 required_capacity=1.354004 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=32 total_expected_recv=246089 total_dropped_tokens=73 drop_rate_pct=0.029664 records_with_drop_pct=3.125000 peak_rank_overflow=73 total_slack_slots=78336 peak_demand=10313 required_capacity=1.258911 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=32 total_expected_recv=258467 total_dropped_tokens=3155 drop_rate_pct=1.220659 records_with_drop_pct=6.250000 peak_rank_overflow=2484 total_slack_slots=69632 peak_demand=12724 required_capacity=1.553223 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=32 total_expected_recv=271675 total_dropped_tokens=3717 drop_rate_pct=1.368179 records_with_drop_pct=12.500000 peak_rank_overflow=2173 total_slack_slots=56704 peak_demand=12413 required_capacity=1.515259 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=32 total_expected_recv=274384 total_dropped_tokens=1647 drop_rate_pct=0.600254 records_with_drop_pct=6.250000 peak_rank_overflow=850 total_slack_slots=52608 peak_demand=11090 required_capacity=1.353760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=32 total_expected_recv=258645 total_dropped_tokens=1494 drop_rate_pct=0.577626 records_with_drop_pct=9.375000 peak_rank_overflow=852 total_slack_slots=67968 peak_demand=11092 required_capacity=1.354004 + [2026-08-30 05:07:57] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 875.9 | throughput per GPU (TFLOP/s/GPU): 380.6 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.104958E+01 | loss scale: 1.0 | grad norm: 37.638 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 10240 | min safe capacity: 1.2500 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=10240 expected_recv=17095 expected_overflow=0 exact_recv_per_rank=[7256, 9003, 8792, 7554, 9005, 5918, 9003, 9005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=10240 expected_recv=16044 expected_overflow=0 exact_recv_per_rank=[7256, 9003, 8792, 7554, 9005, 5918, 9003, 9005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=10240 expected_recv=14833 expected_overflow=0 exact_recv_per_rank=[7256, 9003, 8792, 7554, 9005, 5918, 9003, 9005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=10240 expected_recv=17378 expected_overflow=0 exact_recv_per_rank=[7256, 9003, 8792, 7554, 9005, 5918, 9003, 9005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=10240 expected_recv=17590 expected_overflow=0 exact_recv_per_rank=[7256, 9003, 8792, 7554, 9005, 5918, 9003, 9005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=10240 expected_recv=12953 expected_overflow=0 exact_recv_per_rank=[7256, 9003, 8792, 7554, 9005, 5918, 9003, 9005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=10240 expected_recv=17589 expected_overflow=0 exact_recv_per_rank=[7256, 9003, 8792, 7554, 9005, 5918, 9003, 9005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=10240 expected_recv=17095 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=10240 expected_recv=16044 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=10240 expected_recv=14833 actual_valid_recv=15104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=10240 expected_recv=17378 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=10240 expected_recv=17590 expected_overflow=0 exact_recv_per_rank=[7256, 9003, 8792, 7554, 9005, 5918, 9003, 9005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=10240 expected_recv=17590 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17280 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=10240 expected_recv=12953 actual_valid_recv=13184 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=15104 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17536 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=10240 expected_recv=17589 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17792 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=10240 expected_recv=17590 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=13184 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17792 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17792 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=10240 expected_recv=18126 expected_overflow=0 exact_recv_per_rank=[8878, 8879, 8590, 8878, 7663, 8878, 8878, 4892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=10240 expected_recv=18124 expected_overflow=0 exact_recv_per_rank=[8878, 8879, 8590, 8878, 7663, 8878, 8878, 4892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=10240 expected_recv=17836 expected_overflow=0 exact_recv_per_rank=[8878, 8879, 8590, 8878, 7663, 8878, 8878, 4892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=10240 expected_recv=18124 expected_overflow=0 exact_recv_per_rank=[8878, 8879, 8590, 8878, 7663, 8878, 8878, 4892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=10240 expected_recv=10419 expected_overflow=0 exact_recv_per_rank=[8878, 8879, 8590, 8878, 7663, 8878, 8878, 4892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=10240 expected_recv=18126 actual_valid_recv=18304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=10240 expected_recv=13411 expected_overflow=0 exact_recv_per_rank=[8878, 8879, 8590, 8878, 7663, 8878, 8878, 4892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=10240 expected_recv=18124 expected_overflow=0 exact_recv_per_rank=[8878, 8879, 8590, 8878, 7663, 8878, 8878, 4892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=10240 expected_recv=18124 actual_valid_recv=18304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=10240 expected_recv=18124 actual_valid_recv=18304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=10240 expected_recv=17836 actual_valid_recv=18048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=18304 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=10240 expected_recv=10419 actual_valid_recv=10624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=10240 expected_recv=16908 expected_overflow=0 exact_recv_per_rank=[8878, 8879, 8590, 8878, 7663, 8878, 8878, 4892] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=10240 expected_recv=13411 actual_valid_recv=13568 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=18304 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=18304 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=10240 expected_recv=18124 actual_valid_recv=18304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=18048 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=10624 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=10240 expected_recv=16908 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=13568 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=18304 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=17152 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=10240 expected_recv=18235 expected_overflow=0 exact_recv_per_rank=[3805, 9309, 9308, 5882, 9309, 9308, 9307, 9308] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=10240 expected_recv=11454 expected_overflow=0 exact_recv_per_rank=[3805, 9309, 9308, 5882, 9309, 9308, 9307, 9308] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=10240 expected_recv=10212 expected_overflow=0 exact_recv_per_rank=[3805, 9309, 9308, 5882, 9309, 9308, 9307, 9308] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=10240 expected_recv=18235 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=10240 expected_recv=18234 expected_overflow=0 exact_recv_per_rank=[3805, 9309, 9308, 5882, 9309, 9308, 9307, 9308] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=10240 expected_recv=18234 expected_overflow=0 exact_recv_per_rank=[3805, 9309, 9308, 5882, 9309, 9308, 9307, 9308] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=10240 expected_recv=11454 actual_valid_recv=11648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=18560 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=10240 expected_recv=10212 actual_valid_recv=10368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=10240 expected_recv=18234 expected_overflow=0 exact_recv_per_rank=[3805, 9309, 9308, 5882, 9309, 9308, 9307, 9308] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=10240 expected_recv=18234 actual_valid_recv=18304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=10240 expected_recv=18234 actual_valid_recv=18432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=10240 expected_recv=18234 expected_overflow=0 exact_recv_per_rank=[3805, 9309, 9308, 5882, 9309, 9308, 9307, 9308] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=10240 expected_recv=18235 expected_overflow=0 exact_recv_per_rank=[3805, 9309, 9308, 5882, 9309, 9308, 9307, 9308] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=10368 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=18304 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=10240 expected_recv=18234 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=18432 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=10240 expected_recv=18234 actual_valid_recv=18304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=10240 expected_recv=18235 actual_valid_recv=18304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=18560 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=18304 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=18304 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=10240 expected_recv=17619 expected_overflow=0 exact_recv_per_rank=[7327, 8764, 8209, 8509, 6435, 8764, 8764, 8764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=10240 expected_recv=17619 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=10240 expected_recv=15653 expected_overflow=0 exact_recv_per_rank=[7327, 8764, 8209, 8509, 6435, 8764, 8764, 8764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17920 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=10240 expected_recv=14148 expected_overflow=0 exact_recv_per_rank=[7327, 8764, 8209, 8509, 6435, 8764, 8764, 8764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=10240 expected_recv=15653 actual_valid_recv=15872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=10240 expected_recv=17066 expected_overflow=0 exact_recv_per_rank=[7327, 8764, 8209, 8509, 6435, 8764, 8764, 8764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=10240 expected_recv=17621 expected_overflow=0 exact_recv_per_rank=[7327, 8764, 8209, 8509, 6435, 8764, 8764, 8764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=10240 expected_recv=14148 actual_valid_recv=14464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=15872 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=10240 expected_recv=13980 expected_overflow=0 exact_recv_per_rank=[7327, 8764, 8209, 8509, 6435, 8764, 8764, 8764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=10240 expected_recv=17620 expected_overflow=0 exact_recv_per_rank=[7327, 8764, 8209, 8509, 6435, 8764, 8764, 8764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=10240 expected_recv=17066 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=10240 expected_recv=13980 actual_valid_recv=14208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=14464 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=10240 expected_recv=17621 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17280 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=10240 expected_recv=17620 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=14208 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17792 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=10240 expected_recv=16501 expected_overflow=0 exact_recv_per_rank=[7088, 7546, 8524, 8524, 8525, 8525, 8524, 8280] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17792 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=10240 expected_recv=16501 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=10240 expected_recv=16042 expected_overflow=0 exact_recv_per_rank=[7088, 7546, 8524, 8524, 8525, 8525, 8524, 8280] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=10240 expected_recv=16042 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=10240 expected_recv=11132 expected_overflow=0 exact_recv_per_rank=[7088, 7546, 8524, 8524, 8525, 8525, 8524, 8280] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=10240 expected_recv=17478 expected_overflow=0 exact_recv_per_rank=[7088, 7546, 8524, 8524, 8525, 8525, 8524, 8280] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=10240 expected_recv=17481 expected_overflow=0 exact_recv_per_rank=[7088, 7546, 8524, 8524, 8525, 8525, 8524, 8280] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=11648 slack_slots=8832 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=10240 expected_recv=11132 actual_valid_recv=11520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=10240 expected_recv=17479 expected_overflow=0 exact_recv_per_rank=[7088, 7546, 8524, 8524, 8525, 8525, 8524, 8280] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=10240 expected_recv=17478 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=10240 expected_recv=17480 expected_overflow=0 exact_recv_per_rank=[7088, 7546, 8524, 8524, 8525, 8525, 8524, 8280] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=10240 expected_recv=17481 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=11520 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=10240 expected_recv=17479 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=17664 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=17792 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=10240 expected_recv=17480 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=10240 expected_recv=16870 expected_overflow=0 exact_recv_per_rank=[8595, 8595, 8594, 7451, 8595, 8596, 6515, 8595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=17664 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=17536 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=10240 expected_recv=16869 expected_overflow=0 exact_recv_per_rank=[8595, 8595, 8594, 7451, 8595, 8596, 6515, 8595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=10240 expected_recv=16870 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=10240 expected_recv=16869 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=10240 expected_recv=16868 expected_overflow=0 exact_recv_per_rank=[8595, 8595, 8594, 7451, 8595, 8596, 6515, 8595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=10240 expected_recv=17365 expected_overflow=0 exact_recv_per_rank=[7327, 8764, 8209, 8509, 6435, 8764, 8764, 8764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=10240 expected_recv=16867 expected_overflow=0 exact_recv_per_rank=[8595, 8595, 8594, 7451, 8595, 8596, 6515, 8595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=10240 expected_recv=16870 expected_overflow=0 exact_recv_per_rank=[8595, 8595, 8594, 7451, 8595, 8596, 6515, 8595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=10240 expected_recv=16868 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=10240 expected_recv=16871 expected_overflow=0 exact_recv_per_rank=[8595, 8595, 8594, 7451, 8595, 8596, 6515, 8595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=10240 expected_recv=14438 expected_overflow=0 exact_recv_per_rank=[8595, 8595, 8594, 7451, 8595, 8596, 6515, 8595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=10240 expected_recv=17365 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=10240 expected_recv=16867 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=10240 expected_recv=16349 expected_overflow=0 exact_recv_per_rank=[8860, 8708, 8860, 8860, 8859, 3668, 8860, 8861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=10240 expected_recv=16870 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=17152 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=10240 expected_recv=16871 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17536 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=10240 expected_recv=14438 actual_valid_recv=14720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=17152 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=10240 expected_recv=16349 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=17152 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=10240 expected_recv=18768 expected_overflow=0 exact_recv_per_rank=[8860, 8708, 8860, 8860, 8859, 3668, 8860, 8861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=14720 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=10240 expected_recv=18768 actual_valid_recv=19072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=19072 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=10240 expected_recv=18768 expected_overflow=0 exact_recv_per_rank=[8860, 8708, 8860, 8860, 8859, 3668, 8860, 8861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=10240 expected_recv=17479 expected_overflow=0 exact_recv_per_rank=[7088, 7546, 8524, 8524, 8525, 8525, 8524, 8280] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=10240 expected_recv=18768 expected_overflow=0 exact_recv_per_rank=[8860, 8708, 8860, 8860, 8859, 3668, 8860, 8861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=10240 expected_recv=18768 actual_valid_recv=18944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=10240 expected_recv=10361 expected_overflow=0 exact_recv_per_rank=[8860, 8708, 8860, 8860, 8859, 3668, 8860, 8861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=10240 expected_recv=17479 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=10240 expected_recv=13574 expected_overflow=0 exact_recv_per_rank=[8860, 8708, 8860, 8860, 8859, 3668, 8860, 8861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=10240 expected_recv=15716 expected_overflow=0 exact_recv_per_rank=[8860, 8708, 8860, 8860, 8859, 3668, 8860, 8861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=10240 expected_recv=18768 actual_valid_recv=18944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=18944 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=17536 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=10240 expected_recv=10361 actual_valid_recv=10496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=10240 expected_recv=13574 actual_valid_recv=13824 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=10240 expected_recv=17474 expected_overflow=0 exact_recv_per_rank=[8582, 8583, 8583, 8268, 8582, 8581, 5775, 8582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=18944 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=10240 expected_recv=15716 actual_valid_recv=16000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=10496 slack_slots=9984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=10240 expected_recv=12750 expected_overflow=0 exact_recv_per_rank=[8582, 8583, 8583, 8268, 8582, 8581, 5775, 8582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=13824 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=10240 expected_recv=17474 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16000 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=10240 expected_recv=12750 actual_valid_recv=12928 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=17792 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=10240 expected_recv=17302 expected_overflow=0 exact_recv_per_rank=[8582, 8583, 8583, 8268, 8582, 8581, 5775, 8582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=12928 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=10240 expected_recv=15419 expected_overflow=0 exact_recv_per_rank=[8595, 8595, 8594, 7451, 8595, 8596, 6515, 8595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=10240 expected_recv=17302 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=10240 expected_recv=17470 expected_overflow=0 exact_recv_per_rank=[8582, 8583, 8583, 8268, 8582, 8581, 5775, 8582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=10240 expected_recv=17474 expected_overflow=0 exact_recv_per_rank=[8582, 8583, 8583, 8268, 8582, 8581, 5775, 8582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=10240 expected_recv=15419 actual_valid_recv=15744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=17536 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=10240 expected_recv=17470 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=15744 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=10240 expected_recv=17474 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=10240 expected_recv=17470 expected_overflow=0 exact_recv_per_rank=[8582, 8583, 8583, 8268, 8582, 8581, 5775, 8582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=10240 expected_recv=14666 expected_overflow=0 exact_recv_per_rank=[8582, 8583, 8583, 8268, 8582, 8581, 5775, 8582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=17664 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=17664 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=10240 expected_recv=19738 expected_overflow=309 exact_recv_per_rank=[3351, 10549, 11547, 923, 11547, 11547, 4525, 11547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=10240 expected_recv=17470 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=10240 expected_recv=14666 actual_valid_recv=14848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=10240 expected_recv=9679 expected_overflow=0 exact_recv_per_rank=[3351, 10549, 11547, 923, 11547, 11547, 4525, 11547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=10240 expected_recv=19738 actual_valid_recv=19456 observed_loss=309 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=17664 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=14848 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=10240 expected_recv=9679 actual_valid_recv=9856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=19456 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=10240 expected_recv=20737 expected_overflow=1307 exact_recv_per_rank=[3351, 10549, 11547, 923, 11547, 11547, 4525, 11547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=10240 expected_recv=18768 expected_overflow=0 exact_recv_per_rank=[8860, 8708, 8860, 8860, 8859, 3668, 8860, 8861] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=9856 slack_slots=10624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=10240 expected_recv=20737 actual_valid_recv=19456 observed_loss=1307 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=10240 expected_recv=18768 actual_valid_recv=19200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=10240 expected_recv=20737 expected_overflow=1307 exact_recv_per_rank=[3351, 10549, 11547, 923, 11547, 11547, 4525, 11547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=19456 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=10240 expected_recv=20735 expected_overflow=1307 exact_recv_per_rank=[3351, 10549, 11547, 923, 11547, 11547, 4525, 11547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=19200 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=10240 expected_recv=20737 actual_valid_recv=19456 observed_loss=1307 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=10240 expected_recv=13714 expected_overflow=0 exact_recv_per_rank=[3351, 10549, 11547, 923, 11547, 11547, 4525, 11547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=10240 expected_recv=20735 actual_valid_recv=19584 observed_loss=1307 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=10240 expected_recv=15621 expected_overflow=1307 exact_recv_per_rank=[3351, 10549, 11547, 923, 11547, 11547, 4525, 11547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=19456 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=10240 expected_recv=6101 expected_overflow=0 exact_recv_per_rank=[6364, 2569, 6772, 9967, 9966, 9966, 9966, 9966] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=10240 expected_recv=13714 actual_valid_recv=13952 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=19584 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=10240 expected_recv=17018 expected_overflow=414 exact_recv_per_rank=[6364, 2569, 6772, 9967, 9966, 9966, 9966, 9966] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=10240 expected_recv=15621 actual_valid_recv=14464 observed_loss=1307 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=10240 expected_recv=6101 actual_valid_recv=6272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=13952 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=14464 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=10240 expected_recv=17018 actual_valid_recv=16768 observed_loss=414 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=10240 expected_recv=20621 expected_overflow=415 exact_recv_per_rank=[6364, 2569, 6772, 9967, 9966, 9966, 9966, 9966] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=6272 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=10240 expected_recv=16466 expected_overflow=0 exact_recv_per_rank=[8582, 8583, 8583, 8268, 8582, 8581, 5775, 8582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=10240 expected_recv=20621 actual_valid_recv=20224 observed_loss=415 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=10240 expected_recv=16466 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=10240 expected_recv=20621 expected_overflow=415 exact_recv_per_rank=[6364, 2569, 6772, 9967, 9966, 9966, 9966, 9966] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=20224 slack_slots=256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=10240 expected_recv=15379 expected_overflow=0 exact_recv_per_rank=[6364, 2569, 6772, 9967, 9966, 9966, 9966, 9966] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=10240 expected_recv=20621 actual_valid_recv=20224 observed_loss=415 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=10240 expected_recv=10136 expected_overflow=0 exact_recv_per_rank=[6364, 2569, 6772, 9967, 9966, 9966, 9966, 9966] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=10240 expected_recv=20573 expected_overflow=367 exact_recv_per_rank=[6364, 2569, 6772, 9967, 9966, 9966, 9966, 9966] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=20224 slack_slots=256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=10240 expected_recv=15379 actual_valid_recv=15488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=10240 expected_recv=14542 expected_overflow=0 exact_recv_per_rank=[10036, 5655, 7547, 10038, 10037, 10036, 10036, 2151] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=10240 expected_recv=10136 actual_valid_recv=10368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=10240 expected_recv=18924 expected_overflow=0 exact_recv_per_rank=[10036, 5655, 7547, 10038, 10037, 10036, 10036, 2151] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=10240 expected_recv=20573 actual_valid_recv=20352 observed_loss=367 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=15488 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=10368 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=10240 expected_recv=14542 actual_valid_recv=14848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=20352 slack_slots=128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=10240 expected_recv=18924 actual_valid_recv=19328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=10240 expected_recv=10217 expected_overflow=0 exact_recv_per_rank=[10036, 5655, 7547, 10038, 10037, 10036, 10036, 2151] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=14848 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=10240 expected_recv=10111 expected_overflow=0 exact_recv_per_rank=[3351, 10549, 11547, 923, 11547, 11547, 4525, 11547] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=19328 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=10240 expected_recv=10217 actual_valid_recv=10368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=10240 expected_recv=18924 expected_overflow=0 exact_recv_per_rank=[10036, 5655, 7547, 10038, 10037, 10036, 10036, 2151] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=10240 expected_recv=10111 actual_valid_recv=10240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=10368 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=10240 slack_slots=10240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=10240 expected_recv=18924 actual_valid_recv=19072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=10240 expected_recv=11777 expected_overflow=0 exact_recv_per_rank=[10036, 5655, 7547, 10038, 10037, 10036, 10036, 2151] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=10240 expected_recv=18925 expected_overflow=0 exact_recv_per_rank=[10036, 5655, 7547, 10038, 10037, 10036, 10036, 2151] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=10240 expected_recv=18924 expected_overflow=0 exact_recv_per_rank=[10036, 5655, 7547, 10038, 10037, 10036, 10036, 2151] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=19072 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=10240 expected_recv=11777 actual_valid_recv=12032 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=10240 expected_recv=18925 actual_valid_recv=19200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=10240 expected_recv=18924 actual_valid_recv=19072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=10240 expected_recv=16840 expected_overflow=0 exact_recv_per_rank=[8273, 7852, 8274, 8273, 8275, 8123, 8192, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=12032 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=10240 expected_recv=17262 expected_overflow=0 exact_recv_per_rank=[8273, 7852, 8274, 8273, 8275, 8123, 8192, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=19200 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=19072 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=10240 expected_recv=16840 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=10240 expected_recv=13605 expected_overflow=0 exact_recv_per_rank=[8273, 7852, 8274, 8273, 8275, 8123, 8192, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=10240 expected_recv=17262 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=10240 expected_recv=20623 expected_overflow=416 exact_recv_per_rank=[6364, 2569, 6772, 9967, 9966, 9966, 9966, 9966] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=10240 expected_recv=13605 actual_valid_recv=13824 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=10240 expected_recv=14549 expected_overflow=0 exact_recv_per_rank=[8273, 7852, 8274, 8273, 8275, 8123, 8192, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=10240 expected_recv=20623 actual_valid_recv=20224 observed_loss=416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=13824 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=20224 slack_slots=256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=10240 expected_recv=14549 actual_valid_recv=14720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=10240 expected_recv=17263 expected_overflow=0 exact_recv_per_rank=[8273, 7852, 8274, 8273, 8275, 8123, 8192, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=10240 expected_recv=17179 expected_overflow=0 exact_recv_per_rank=[8273, 7852, 8274, 8273, 8275, 8123, 8192, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=14720 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=10240 expected_recv=17112 expected_overflow=0 exact_recv_per_rank=[8273, 7852, 8274, 8273, 8275, 8123, 8192, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=10240 expected_recv=17263 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=10240 expected_recv=17179 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=10240 expected_recv=21216 expected_overflow=2123 exact_recv_per_rank=[3603, 8853, 8853, 8821, 8851, 8851, 8852, 8852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=10240 expected_recv=17112 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=10240 expected_recv=4145 expected_overflow=0 exact_recv_per_rank=[3603, 8853, 8853, 8821, 8851, 8851, 8852, 8852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=17280 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=10240 expected_recv=21216 actual_valid_recv=19200 observed_loss=2123 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=10240 expected_recv=15191 expected_overflow=0 exact_recv_per_rank=[3603, 8853, 8853, 8821, 8851, 8851, 8852, 8852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=10240 expected_recv=4145 actual_valid_recv=4352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=10240 expected_recv=18839 expected_overflow=0 exact_recv_per_rank=[10036, 5655, 7547, 10038, 10037, 10036, 10036, 2151] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=19200 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=10240 expected_recv=15191 actual_valid_recv=15488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=4352 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=10240 expected_recv=21212 expected_overflow=2121 exact_recv_per_rank=[3603, 8853, 8853, 8821, 8851, 8851, 8852, 8852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=10240 expected_recv=18839 actual_valid_recv=18944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=15488 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=10240 expected_recv=21212 actual_valid_recv=19200 observed_loss=2121 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=18944 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=10240 expected_recv=18275 expected_overflow=0 exact_recv_per_rank=[3603, 8853, 8853, 8821, 8851, 8851, 8852, 8852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=19200 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=10240 expected_recv=21215 expected_overflow=2123 exact_recv_per_rank=[3603, 8853, 8853, 8821, 8851, 8851, 8852, 8852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=10240 expected_recv=9153 expected_overflow=0 exact_recv_per_rank=[3603, 8853, 8853, 8821, 8851, 8851, 8852, 8852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=10240 expected_recv=18275 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=10240 expected_recv=21215 actual_valid_recv=19200 observed_loss=2123 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=10240 expected_recv=9153 actual_valid_recv=9344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=10240 expected_recv=16804 expected_overflow=0 exact_recv_per_rank=[8987, 8216, 7291, 8986, 8582, 8987, 8988, 5499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=10240 expected_recv=17576 expected_overflow=0 exact_recv_per_rank=[8987, 8216, 7291, 8986, 8582, 8987, 8988, 5499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=18560 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=10240 expected_recv=11731 expected_overflow=0 exact_recv_per_rank=[8987, 8216, 7291, 8986, 8582, 8987, 8988, 5499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=19200 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=9344 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=10240 expected_recv=16804 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=10240 expected_recv=17576 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=10240 expected_recv=17262 expected_overflow=0 exact_recv_per_rank=[8273, 7852, 8274, 8273, 8275, 8123, 8192, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=10240 expected_recv=11731 actual_valid_recv=11904 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=17792 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=10240 expected_recv=17262 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=11904 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=10240 expected_recv=17169 expected_overflow=0 exact_recv_per_rank=[8987, 8216, 7291, 8986, 8582, 8987, 8988, 5499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=17664 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=10240 expected_recv=17169 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=10240 expected_recv=15513 expected_overflow=0 exact_recv_per_rank=[8987, 8216, 7291, 8986, 8582, 8987, 8988, 5499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=10240 expected_recv=17576 expected_overflow=0 exact_recv_per_rank=[8987, 8216, 7291, 8986, 8582, 8987, 8988, 5499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=10240 expected_recv=17129 expected_overflow=0 exact_recv_per_rank=[8987, 8216, 7291, 8986, 8582, 8987, 8988, 5499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=10240 expected_recv=15513 actual_valid_recv=15616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=10240 expected_recv=10277 expected_overflow=0 exact_recv_per_rank=[11350, 1756, 10288, 11350, 11350, 1574, 11253, 6615] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=10240 expected_recv=17129 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=10240 expected_recv=17576 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=10240 expected_recv=19873 expected_overflow=1110 exact_recv_per_rank=[11350, 1756, 10288, 11350, 11350, 1574, 11253, 6615] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=15616 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=10240 expected_recv=13677 expected_overflow=0 exact_recv_per_rank=[11350, 1756, 10288, 11350, 11350, 1574, 11253, 6615] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=10240 expected_recv=10277 actual_valid_recv=10496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=17792 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=10240 expected_recv=19873 actual_valid_recv=18816 observed_loss=1110 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=10240 expected_recv=20665 expected_overflow=1604 exact_recv_per_rank=[3603, 8853, 8853, 8821, 8851, 8851, 8852, 8852] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=10240 expected_recv=13677 actual_valid_recv=13824 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=10496 slack_slots=9984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=18816 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=10240 expected_recv=19872 expected_overflow=1110 exact_recv_per_rank=[11350, 1756, 10288, 11350, 11350, 1574, 11253, 6615] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=10240 expected_recv=20665 actual_valid_recv=19200 observed_loss=1604 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=13824 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=19200 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=10240 expected_recv=19872 actual_valid_recv=18816 observed_loss=1110 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=10240 expected_recv=18336 expected_overflow=48 exact_recv_per_rank=[11350, 1756, 10288, 11350, 11350, 1574, 11253, 6615] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=18816 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=10240 expected_recv=9388 expected_overflow=0 exact_recv_per_rank=[11350, 1756, 10288, 11350, 11350, 1574, 11253, 6615] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=10240 expected_recv=18336 actual_valid_recv=18432 observed_loss=48 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=10240 expected_recv=19776 expected_overflow=1013 exact_recv_per_rank=[11350, 1756, 10288, 11350, 11350, 1574, 11253, 6615] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8048, 8274, 8274, 8274, 7844, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=10240 expected_recv=16164 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8048, 8274, 8274, 8274, 7844, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=18432 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=10240 expected_recv=9388 actual_valid_recv=9728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=10240 expected_recv=19776 actual_valid_recv=18944 observed_loss=1013 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8048, 8274, 8274, 8274, 7844, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=10240 expected_recv=17574 expected_overflow=0 exact_recv_per_rank=[8987, 8216, 7291, 8986, 8582, 8987, 8988, 5499] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=9728 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=18944 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=10240 expected_recv=16164 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=10240 expected_recv=16275 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8048, 8274, 8274, 8274, 7844, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=10240 expected_recv=17574 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=17792 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=10240 expected_recv=16275 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=10240 expected_recv=16322 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8048, 8274, 8274, 8274, 7844, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=10240 expected_recv=16322 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8048, 8274, 8274, 8274, 7844, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=10240 expected_recv=16118 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8048, 8274, 8274, 8274, 7844, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[649, 4960, 8792, 2874, 13048, 5918, 13683, 15612] avg=8192.0000 max=15612 max_rank=7 min=649 min_rank=0 imbalance_ratio=1.905762 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=10240 expected_recv=19873 expected_overflow=1110 exact_recv_per_rank=[11350, 1756, 10288, 11350, 11350, 1574, 11253, 6615] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=10240 expected_recv=16118 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[649, 4960, 8792, 2874, 13048, 5918, 13683, 15612] avg=8192.0000 max=15612 max_rank=7 min=649 min_rank=0 imbalance_ratio=1.905762 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[649, 4960, 8792, 2874, 13048, 5918, 13683, 15612] avg=8192.0000 max=15612 max_rank=7 min=649 min_rank=0 imbalance_ratio=1.905762 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=10240 expected_recv=19873 actual_valid_recv=18816 observed_loss=1110 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[649, 4960, 8792, 2874, 13048, 5918, 13683, 15612] avg=8192.0000 max=15612 max_rank=7 min=649 min_rank=0 imbalance_ratio=1.905762 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=18816 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[649, 4960, 8792, 2874, 13048, 5918, 13683, 15612] avg=8192.0000 max=15612 max_rank=7 min=649 min_rank=0 imbalance_ratio=1.905762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[7256, 9003, 8792, 7554, 9005, 5918, 9003, 9005] avg=8192.0000 max=9005 max_rank=4 min=5918 min_rank=5 imbalance_ratio=1.099243 max_load_reduction=6607 max_load_reduction_pct=0.423200 moved_to_replicas=32638 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[7256, 9003, 8792, 7554, 9005, 5918, 9003, 9005] avg=8192.0000 max=9005 max_rank=4 min=5918 min_rank=5 imbalance_ratio=1.099243 max_load_reduction=6607 max_load_reduction_pct=0.423200 moved_to_replicas=32638 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[7256, 9003, 8792, 7554, 9005, 5918, 9003, 9005] avg=8192.0000 max=9005 max_rank=4 min=5918 min_rank=5 imbalance_ratio=1.099243 max_load_reduction=6607 max_load_reduction_pct=0.423200 moved_to_replicas=32638 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[7256, 9003, 8792, 7554, 9005, 5918, 9003, 9005] avg=8192.0000 max=9005 max_rank=4 min=5918 min_rank=5 imbalance_ratio=1.099243 max_load_reduction=6607 max_load_reduction_pct=0.423200 moved_to_replicas=32638 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[649, 4960, 8792, 2874, 13048, 5918, 13683, 15612] avg=8192.0000 max=15612 max_rank=7 min=649 min_rank=0 imbalance_ratio=1.905762 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8048, 8274, 8274, 8274, 7844, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[649, 4960, 8792, 2874, 13048, 5918, 13683, 15612] avg=8192.0000 max=15612 max_rank=7 min=649 min_rank=0 imbalance_ratio=1.905762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[7256, 9003, 8792, 7554, 9005, 5918, 9003, 9005] avg=8192.0000 max=9005 max_rank=4 min=5918 min_rank=5 imbalance_ratio=1.099243 max_load_reduction=6607 max_load_reduction_pct=0.423200 moved_to_replicas=32638 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[15298, 11129, 2170, 30154, 1972, 1587, 584, 2642] avg=8192.0000 max=30154 max_rank=3 min=584 min_rank=6 imbalance_ratio=3.680908 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[15298, 11129, 2170, 30154, 1972, 1587, 584, 2642] avg=8192.0000 max=30154 max_rank=3 min=584 min_rank=6 imbalance_ratio=3.680908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[15298, 11129, 2170, 30154, 1972, 1587, 584, 2642] avg=8192.0000 max=30154 max_rank=3 min=584 min_rank=6 imbalance_ratio=3.680908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[7256, 9003, 8792, 7554, 9005, 5918, 9003, 9005] avg=8192.0000 max=9005 max_rank=4 min=5918 min_rank=5 imbalance_ratio=1.099243 max_load_reduction=6607 max_load_reduction_pct=0.423200 moved_to_replicas=32638 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[7256, 9003, 8792, 7554, 9005, 5918, 9003, 9005] avg=8192.0000 max=9005 max_rank=4 min=5918 min_rank=5 imbalance_ratio=1.099243 max_load_reduction=6607 max_load_reduction_pct=0.423200 moved_to_replicas=32638 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[15298, 11129, 2170, 30154, 1972, 1587, 584, 2642] avg=8192.0000 max=30154 max_rank=3 min=584 min_rank=6 imbalance_ratio=3.680908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[8878, 8879, 8590, 8878, 7663, 8878, 8878, 4892] avg=8192.0000 max=8879 max_rank=1 min=4892 min_rank=7 imbalance_ratio=1.083862 max_load_reduction=21275 max_load_reduction_pct=0.705545 moved_to_replicas=60308 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[8878, 8879, 8590, 8878, 7663, 8878, 8878, 4892] avg=8192.0000 max=8879 max_rank=1 min=4892 min_rank=7 imbalance_ratio=1.083862 max_load_reduction=21275 max_load_reduction_pct=0.705545 moved_to_replicas=60308 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[15298, 11129, 2170, 30154, 1972, 1587, 584, 2642] avg=8192.0000 max=30154 max_rank=3 min=584 min_rank=6 imbalance_ratio=3.680908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[8878, 8879, 8590, 8878, 7663, 8878, 8878, 4892] avg=8192.0000 max=8879 max_rank=1 min=4892 min_rank=7 imbalance_ratio=1.083862 max_load_reduction=21275 max_load_reduction_pct=0.705545 moved_to_replicas=60308 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[8878, 8879, 8590, 8878, 7663, 8878, 8878, 4892] avg=8192.0000 max=8879 max_rank=1 min=4892 min_rank=7 imbalance_ratio=1.083862 max_load_reduction=21275 max_load_reduction_pct=0.705545 moved_to_replicas=60308 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[649, 4960, 8792, 2874, 13048, 5918, 13683, 15612] avg=8192.0000 max=15612 max_rank=7 min=649 min_rank=0 imbalance_ratio=1.905762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[15298, 11129, 2170, 30154, 1972, 1587, 584, 2642] avg=8192.0000 max=30154 max_rank=3 min=584 min_rank=6 imbalance_ratio=3.680908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[15298, 11129, 2170, 30154, 1972, 1587, 584, 2642] avg=8192.0000 max=30154 max_rank=3 min=584 min_rank=6 imbalance_ratio=3.680908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[8878, 8879, 8590, 8878, 7663, 8878, 8878, 4892] avg=8192.0000 max=8879 max_rank=1 min=4892 min_rank=7 imbalance_ratio=1.083862 max_load_reduction=21275 max_load_reduction_pct=0.705545 moved_to_replicas=60308 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[2161, 2108, 1386, 1891, 10953, 1484, 16508, 29045] avg=8192.0000 max=29045 max_rank=7 min=1386 min_rank=2 imbalance_ratio=3.545532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[2161, 2108, 1386, 1891, 10953, 1484, 16508, 29045] avg=8192.0000 max=29045 max_rank=7 min=1386 min_rank=2 imbalance_ratio=3.545532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[7256, 9003, 8792, 7554, 9005, 5918, 9003, 9005] avg=8192.0000 max=9005 max_rank=4 min=5918 min_rank=5 imbalance_ratio=1.099243 max_load_reduction=6607 max_load_reduction_pct=0.423200 moved_to_replicas=32638 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[8878, 8879, 8590, 8878, 7663, 8878, 8878, 4892] avg=8192.0000 max=8879 max_rank=1 min=4892 min_rank=7 imbalance_ratio=1.083862 max_load_reduction=21275 max_load_reduction_pct=0.705545 moved_to_replicas=60308 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[2161, 2108, 1386, 1891, 10953, 1484, 16508, 29045] avg=8192.0000 max=29045 max_rank=7 min=1386 min_rank=2 imbalance_ratio=3.545532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[8878, 8879, 8590, 8878, 7663, 8878, 8878, 4892] avg=8192.0000 max=8879 max_rank=1 min=4892 min_rank=7 imbalance_ratio=1.083862 max_load_reduction=21275 max_load_reduction_pct=0.705545 moved_to_replicas=60308 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[3805, 9309, 9308, 5882, 9309, 9308, 9307, 9308] avg=8192.0000 max=9309 max_rank=1 min=3805 min_rank=0 imbalance_ratio=1.136353 max_load_reduction=19736 max_load_reduction_pct=0.679497 moved_to_replicas=58621 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[2161, 2108, 1386, 1891, 10953, 1484, 16508, 29045] avg=8192.0000 max=29045 max_rank=7 min=1386 min_rank=2 imbalance_ratio=3.545532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[3805, 9309, 9308, 5882, 9309, 9308, 9307, 9308] avg=8192.0000 max=9309 max_rank=1 min=3805 min_rank=0 imbalance_ratio=1.136353 max_load_reduction=19736 max_load_reduction_pct=0.679497 moved_to_replicas=58621 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[2161, 2108, 1386, 1891, 10953, 1484, 16508, 29045] avg=8192.0000 max=29045 max_rank=7 min=1386 min_rank=2 imbalance_ratio=3.545532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[3805, 9309, 9308, 5882, 9309, 9308, 9307, 9308] avg=8192.0000 max=9309 max_rank=1 min=3805 min_rank=0 imbalance_ratio=1.136353 max_load_reduction=19736 max_load_reduction_pct=0.679497 moved_to_replicas=58621 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[15298, 11129, 2170, 30154, 1972, 1587, 584, 2642] avg=8192.0000 max=30154 max_rank=3 min=584 min_rank=6 imbalance_ratio=3.680908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[2161, 2108, 1386, 1891, 10953, 1484, 16508, 29045] avg=8192.0000 max=29045 max_rank=7 min=1386 min_rank=2 imbalance_ratio=3.545532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[3805, 9309, 9308, 5882, 9309, 9308, 9307, 9308] avg=8192.0000 max=9309 max_rank=1 min=3805 min_rank=0 imbalance_ratio=1.136353 max_load_reduction=19736 max_load_reduction_pct=0.679497 moved_to_replicas=58621 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[2161, 2108, 1386, 1891, 10953, 1484, 16508, 29045] avg=8192.0000 max=29045 max_rank=7 min=1386 min_rank=2 imbalance_ratio=3.545532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[7327, 798, 8209, 833, 5636, 17013, 24406, 1314] avg=8192.0000 max=24406 max_rank=6 min=798 min_rank=1 imbalance_ratio=2.979248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[3805, 9309, 9308, 5882, 9309, 9308, 9307, 9308] avg=8192.0000 max=9309 max_rank=1 min=3805 min_rank=0 imbalance_ratio=1.136353 max_load_reduction=19736 max_load_reduction_pct=0.679497 moved_to_replicas=58621 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[7327, 798, 8209, 833, 5636, 17013, 24406, 1314] avg=8192.0000 max=24406 max_rank=6 min=798 min_rank=1 imbalance_ratio=2.979248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[8878, 8879, 8590, 8878, 7663, 8878, 8878, 4892] avg=8192.0000 max=8879 max_rank=1 min=4892 min_rank=7 imbalance_ratio=1.083862 max_load_reduction=21275 max_load_reduction_pct=0.705545 moved_to_replicas=60308 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[3805, 9309, 9308, 5882, 9309, 9308, 9307, 9308] avg=8192.0000 max=9309 max_rank=1 min=3805 min_rank=0 imbalance_ratio=1.136353 max_load_reduction=19736 max_load_reduction_pct=0.679497 moved_to_replicas=58621 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[7327, 798, 8209, 833, 5636, 17013, 24406, 1314] avg=8192.0000 max=24406 max_rank=6 min=798 min_rank=1 imbalance_ratio=2.979248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[7327, 8764, 8209, 8509, 6435, 8764, 8764, 8764] avg=8192.0000 max=8764 max_rank=1 min=6435 min_rank=4 imbalance_ratio=1.069824 max_load_reduction=15642 max_load_reduction_pct=0.640908 moved_to_replicas=52421 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[3805, 9309, 9308, 5882, 9309, 9308, 9307, 9308] avg=8192.0000 max=9309 max_rank=1 min=3805 min_rank=0 imbalance_ratio=1.136353 max_load_reduction=19736 max_load_reduction_pct=0.679497 moved_to_replicas=58621 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[7327, 798, 8209, 833, 5636, 17013, 24406, 1314] avg=8192.0000 max=24406 max_rank=6 min=798 min_rank=1 imbalance_ratio=2.979248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[7327, 8764, 8209, 8509, 6435, 8764, 8764, 8764] avg=8192.0000 max=8764 max_rank=1 min=6435 min_rank=4 imbalance_ratio=1.069824 max_load_reduction=15642 max_load_reduction_pct=0.640908 moved_to_replicas=52421 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[7327, 798, 8209, 833, 5636, 17013, 24406, 1314] avg=8192.0000 max=24406 max_rank=6 min=798 min_rank=1 imbalance_ratio=2.979248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[7327, 8764, 8209, 8509, 6435, 8764, 8764, 8764] avg=8192.0000 max=8764 max_rank=1 min=6435 min_rank=4 imbalance_ratio=1.069824 max_load_reduction=15642 max_load_reduction_pct=0.640908 moved_to_replicas=52421 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[2161, 2108, 1386, 1891, 10953, 1484, 16508, 29045] avg=8192.0000 max=29045 max_rank=7 min=1386 min_rank=2 imbalance_ratio=3.545532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[7327, 8764, 8209, 8509, 6435, 8764, 8764, 8764] avg=8192.0000 max=8764 max_rank=1 min=6435 min_rank=4 imbalance_ratio=1.069824 max_load_reduction=15642 max_load_reduction_pct=0.640908 moved_to_replicas=52421 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[7327, 798, 8209, 833, 5636, 17013, 24406, 1314] avg=8192.0000 max=24406 max_rank=6 min=798 min_rank=1 imbalance_ratio=2.979248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[1005, 7499, 8571, 24967, 160, 210, 22922, 202] avg=8192.0000 max=24967 max_rank=3 min=160 min_rank=4 imbalance_ratio=3.047729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[7327, 798, 8209, 833, 5636, 17013, 24406, 1314] avg=8192.0000 max=24406 max_rank=6 min=798 min_rank=1 imbalance_ratio=2.979248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[7327, 8764, 8209, 8509, 6435, 8764, 8764, 8764] avg=8192.0000 max=8764 max_rank=1 min=6435 min_rank=4 imbalance_ratio=1.069824 max_load_reduction=15642 max_load_reduction_pct=0.640908 moved_to_replicas=52421 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[1005, 7499, 8571, 24967, 160, 210, 22922, 202] avg=8192.0000 max=24967 max_rank=3 min=160 min_rank=4 imbalance_ratio=3.047729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[3805, 9309, 9308, 5882, 9309, 9308, 9307, 9308] avg=8192.0000 max=9309 max_rank=1 min=3805 min_rank=0 imbalance_ratio=1.136353 max_load_reduction=19736 max_load_reduction_pct=0.679497 moved_to_replicas=58621 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[7327, 8764, 8209, 8509, 6435, 8764, 8764, 8764] avg=8192.0000 max=8764 max_rank=1 min=6435 min_rank=4 imbalance_ratio=1.069824 max_load_reduction=15642 max_load_reduction_pct=0.640908 moved_to_replicas=52421 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[1005, 7499, 8571, 24967, 160, 210, 22922, 202] avg=8192.0000 max=24967 max_rank=3 min=160 min_rank=4 imbalance_ratio=3.047729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[1005, 7499, 8571, 24967, 160, 210, 22922, 202] avg=8192.0000 max=24967 max_rank=3 min=160 min_rank=4 imbalance_ratio=3.047729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[7088, 7546, 8524, 8524, 8525, 8525, 8524, 8280] avg=8192.0000 max=8525 max_rank=4 min=7088 min_rank=0 imbalance_ratio=1.040649 max_load_reduction=16442 max_load_reduction_pct=0.658549 moved_to_replicas=66132 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[7327, 8764, 8209, 8509, 6435, 8764, 8764, 8764] avg=8192.0000 max=8764 max_rank=1 min=6435 min_rank=4 imbalance_ratio=1.069824 max_load_reduction=15642 max_load_reduction_pct=0.640908 moved_to_replicas=52421 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[7088, 7546, 8524, 8524, 8525, 8525, 8524, 8280] avg=8192.0000 max=8525 max_rank=4 min=7088 min_rank=0 imbalance_ratio=1.040649 max_load_reduction=16442 max_load_reduction_pct=0.658549 moved_to_replicas=66132 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[7088, 7546, 8524, 8524, 8525, 8525, 8524, 8280] avg=8192.0000 max=8525 max_rank=4 min=7088 min_rank=0 imbalance_ratio=1.040649 max_load_reduction=16442 max_load_reduction_pct=0.658549 moved_to_replicas=66132 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[1005, 7499, 8571, 24967, 160, 210, 22922, 202] avg=8192.0000 max=24967 max_rank=3 min=160 min_rank=4 imbalance_ratio=3.047729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[7327, 798, 8209, 833, 5636, 17013, 24406, 1314] avg=8192.0000 max=24406 max_rank=6 min=798 min_rank=1 imbalance_ratio=2.979248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[7088, 7546, 8524, 8524, 8525, 8525, 8524, 8280] avg=8192.0000 max=8525 max_rank=4 min=7088 min_rank=0 imbalance_ratio=1.040649 max_load_reduction=16442 max_load_reduction_pct=0.658549 moved_to_replicas=66132 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[1005, 7499, 8571, 24967, 160, 210, 22922, 202] avg=8192.0000 max=24967 max_rank=3 min=160 min_rank=4 imbalance_ratio=3.047729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[1005, 7499, 8571, 24967, 160, 210, 22922, 202] avg=8192.0000 max=24967 max_rank=3 min=160 min_rank=4 imbalance_ratio=3.047729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[474, 32075, 564, 1547, 35, 30651, 190, 0] avg=8192.0000 max=32075 max_rank=1 min=0 min_rank=7 imbalance_ratio=3.915405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[7327, 8764, 8209, 8509, 6435, 8764, 8764, 8764] avg=8192.0000 max=8764 max_rank=1 min=6435 min_rank=4 imbalance_ratio=1.069824 max_load_reduction=15642 max_load_reduction_pct=0.640908 moved_to_replicas=52421 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[474, 32075, 564, 1547, 35, 30651, 190, 0] avg=8192.0000 max=32075 max_rank=1 min=0 min_rank=7 imbalance_ratio=3.915405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[7088, 7546, 8524, 8524, 8525, 8525, 8524, 8280] avg=8192.0000 max=8525 max_rank=4 min=7088 min_rank=0 imbalance_ratio=1.040649 max_load_reduction=16442 max_load_reduction_pct=0.658549 moved_to_replicas=66132 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[474, 32075, 564, 1547, 35, 30651, 190, 0] avg=8192.0000 max=32075 max_rank=1 min=0 min_rank=7 imbalance_ratio=3.915405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[7088, 7546, 8524, 8524, 8525, 8525, 8524, 8280] avg=8192.0000 max=8525 max_rank=4 min=7088 min_rank=0 imbalance_ratio=1.040649 max_load_reduction=16442 max_load_reduction_pct=0.658549 moved_to_replicas=66132 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[474, 32075, 564, 1547, 35, 30651, 190, 0] avg=8192.0000 max=32075 max_rank=1 min=0 min_rank=7 imbalance_ratio=3.915405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[8595, 8595, 8594, 7451, 8595, 8596, 6515, 8595] avg=8192.0000 max=8596 max_rank=5 min=6515 min_rank=6 imbalance_ratio=1.049316 max_load_reduction=23479 max_load_reduction_pct=0.732003 moved_to_replicas=93381 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[7088, 7546, 8524, 8524, 8525, 8525, 8524, 8280] avg=8192.0000 max=8525 max_rank=4 min=7088 min_rank=0 imbalance_ratio=1.040649 max_load_reduction=16442 max_load_reduction_pct=0.658549 moved_to_replicas=66132 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[8595, 8595, 8594, 7451, 8595, 8596, 6515, 8595] avg=8192.0000 max=8596 max_rank=5 min=6515 min_rank=6 imbalance_ratio=1.049316 max_load_reduction=23479 max_load_reduction_pct=0.732003 moved_to_replicas=93381 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[1005, 7499, 8571, 24967, 160, 210, 22922, 202] avg=8192.0000 max=24967 max_rank=3 min=160 min_rank=4 imbalance_ratio=3.047729 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[8595, 8595, 8594, 7451, 8595, 8596, 6515, 8595] avg=8192.0000 max=8596 max_rank=5 min=6515 min_rank=6 imbalance_ratio=1.049316 max_load_reduction=23479 max_load_reduction_pct=0.732003 moved_to_replicas=93381 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[474, 32075, 564, 1547, 35, 30651, 190, 0] avg=8192.0000 max=32075 max_rank=1 min=0 min_rank=7 imbalance_ratio=3.915405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[8595, 8595, 8594, 7451, 8595, 8596, 6515, 8595] avg=8192.0000 max=8596 max_rank=5 min=6515 min_rank=6 imbalance_ratio=1.049316 max_load_reduction=23479 max_load_reduction_pct=0.732003 moved_to_replicas=93381 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[474, 32075, 564, 1547, 35, 30651, 190, 0] avg=8192.0000 max=32075 max_rank=1 min=0 min_rank=7 imbalance_ratio=3.915405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[38, 7204, 29281, 393, 1225, 536, 571, 26288] avg=8192.0000 max=29281 max_rank=2 min=38 min_rank=0 imbalance_ratio=3.574341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[7088, 7546, 8524, 8524, 8525, 8525, 8524, 8280] avg=8192.0000 max=8525 max_rank=4 min=7088 min_rank=0 imbalance_ratio=1.040649 max_load_reduction=16442 max_load_reduction_pct=0.658549 moved_to_replicas=66132 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[474, 32075, 564, 1547, 35, 30651, 190, 0] avg=8192.0000 max=32075 max_rank=1 min=0 min_rank=7 imbalance_ratio=3.915405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[38, 7204, 29281, 393, 1225, 536, 571, 26288] avg=8192.0000 max=29281 max_rank=2 min=38 min_rank=0 imbalance_ratio=3.574341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[8595, 8595, 8594, 7451, 8595, 8596, 6515, 8595] avg=8192.0000 max=8596 max_rank=5 min=6515 min_rank=6 imbalance_ratio=1.049316 max_load_reduction=23479 max_load_reduction_pct=0.732003 moved_to_replicas=93381 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[8595, 8595, 8594, 7451, 8595, 8596, 6515, 8595] avg=8192.0000 max=8596 max_rank=5 min=6515 min_rank=6 imbalance_ratio=1.049316 max_load_reduction=23479 max_load_reduction_pct=0.732003 moved_to_replicas=93381 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[38, 7204, 29281, 393, 1225, 536, 571, 26288] avg=8192.0000 max=29281 max_rank=2 min=38 min_rank=0 imbalance_ratio=3.574341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[8860, 8708, 8860, 8860, 8859, 3668, 8860, 8861] avg=8192.0000 max=8861 max_rank=7 min=3668 min_rank=5 imbalance_ratio=1.081665 max_load_reduction=20420 max_load_reduction_pct=0.697381 moved_to_replicas=73699 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[38, 7204, 29281, 393, 1225, 536, 571, 26288] avg=8192.0000 max=29281 max_rank=2 min=38 min_rank=0 imbalance_ratio=3.574341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[8595, 8595, 8594, 7451, 8595, 8596, 6515, 8595] avg=8192.0000 max=8596 max_rank=5 min=6515 min_rank=6 imbalance_ratio=1.049316 max_load_reduction=23479 max_load_reduction_pct=0.732003 moved_to_replicas=93381 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[8860, 8708, 8860, 8860, 8859, 3668, 8860, 8861] avg=8192.0000 max=8861 max_rank=7 min=3668 min_rank=5 imbalance_ratio=1.081665 max_load_reduction=20420 max_load_reduction_pct=0.697381 moved_to_replicas=73699 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[474, 32075, 564, 1547, 35, 30651, 190, 0] avg=8192.0000 max=32075 max_rank=1 min=0 min_rank=7 imbalance_ratio=3.915405 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[8860, 8708, 8860, 8860, 8859, 3668, 8860, 8861] avg=8192.0000 max=8861 max_rank=7 min=3668 min_rank=5 imbalance_ratio=1.081665 max_load_reduction=20420 max_load_reduction_pct=0.697381 moved_to_replicas=73699 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[38, 7204, 29281, 393, 1225, 536, 571, 26288] avg=8192.0000 max=29281 max_rank=2 min=38 min_rank=0 imbalance_ratio=3.574341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[38, 7204, 29281, 393, 1225, 536, 571, 26288] avg=8192.0000 max=29281 max_rank=2 min=38 min_rank=0 imbalance_ratio=3.574341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[8860, 8708, 8860, 8860, 8859, 3668, 8860, 8861] avg=8192.0000 max=8861 max_rank=7 min=3668 min_rank=5 imbalance_ratio=1.081665 max_load_reduction=20420 max_load_reduction_pct=0.697381 moved_to_replicas=73699 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[239, 50, 20848, 6128, 30788, 3199, 445, 3839] avg=8192.0000 max=30788 max_rank=4 min=50 min_rank=1 imbalance_ratio=3.758301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[8595, 8595, 8594, 7451, 8595, 8596, 6515, 8595] avg=8192.0000 max=8596 max_rank=5 min=6515 min_rank=6 imbalance_ratio=1.049316 max_load_reduction=23479 max_load_reduction_pct=0.732003 moved_to_replicas=93381 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[38, 7204, 29281, 393, 1225, 536, 571, 26288] avg=8192.0000 max=29281 max_rank=2 min=38 min_rank=0 imbalance_ratio=3.574341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[8860, 8708, 8860, 8860, 8859, 3668, 8860, 8861] avg=8192.0000 max=8861 max_rank=7 min=3668 min_rank=5 imbalance_ratio=1.081665 max_load_reduction=20420 max_load_reduction_pct=0.697381 moved_to_replicas=73699 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[8860, 8708, 8860, 8860, 8859, 3668, 8860, 8861] avg=8192.0000 max=8861 max_rank=7 min=3668 min_rank=5 imbalance_ratio=1.081665 max_load_reduction=20420 max_load_reduction_pct=0.697381 moved_to_replicas=73699 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[239, 50, 20848, 6128, 30788, 3199, 445, 3839] avg=8192.0000 max=30788 max_rank=4 min=50 min_rank=1 imbalance_ratio=3.758301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[239, 50, 20848, 6128, 30788, 3199, 445, 3839] avg=8192.0000 max=30788 max_rank=4 min=50 min_rank=1 imbalance_ratio=3.758301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[8582, 8583, 8583, 8268, 8582, 8581, 5775, 8582] avg=8192.0000 max=8583 max_rank=1 min=5775 min_rank=6 imbalance_ratio=1.047729 max_load_reduction=22205 max_load_reduction_pct=0.721223 moved_to_replicas=71568 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[239, 50, 20848, 6128, 30788, 3199, 445, 3839] avg=8192.0000 max=30788 max_rank=4 min=50 min_rank=1 imbalance_ratio=3.758301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[8860, 8708, 8860, 8860, 8859, 3668, 8860, 8861] avg=8192.0000 max=8861 max_rank=7 min=3668 min_rank=5 imbalance_ratio=1.081665 max_load_reduction=20420 max_load_reduction_pct=0.697381 moved_to_replicas=73699 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[38, 7204, 29281, 393, 1225, 536, 571, 26288] avg=8192.0000 max=29281 max_rank=2 min=38 min_rank=0 imbalance_ratio=3.574341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[8582, 8583, 8583, 8268, 8582, 8581, 5775, 8582] avg=8192.0000 max=8583 max_rank=1 min=5775 min_rank=6 imbalance_ratio=1.047729 max_load_reduction=22205 max_load_reduction_pct=0.721223 moved_to_replicas=71568 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[8582, 8583, 8583, 8268, 8582, 8581, 5775, 8582] avg=8192.0000 max=8583 max_rank=1 min=5775 min_rank=6 imbalance_ratio=1.047729 max_load_reduction=22205 max_load_reduction_pct=0.721223 moved_to_replicas=71568 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[8582, 8583, 8583, 8268, 8582, 8581, 5775, 8582] avg=8192.0000 max=8583 max_rank=1 min=5775 min_rank=6 imbalance_ratio=1.047729 max_load_reduction=22205 max_load_reduction_pct=0.721223 moved_to_replicas=71568 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[239, 50, 20848, 6128, 30788, 3199, 445, 3839] avg=8192.0000 max=30788 max_rank=4 min=50 min_rank=1 imbalance_ratio=3.758301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[239, 50, 20848, 6128, 30788, 3199, 445, 3839] avg=8192.0000 max=30788 max_rank=4 min=50 min_rank=1 imbalance_ratio=3.758301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[8860, 8708, 8860, 8860, 8859, 3668, 8860, 8861] avg=8192.0000 max=8861 max_rank=7 min=3668 min_rank=5 imbalance_ratio=1.081665 max_load_reduction=20420 max_load_reduction_pct=0.697381 moved_to_replicas=73699 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[3351, 10549, 1, 923, 27535, 602, 83, 22492] avg=8192.0000 max=27535 max_rank=4 min=1 min_rank=2 imbalance_ratio=3.361206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[8582, 8583, 8583, 8268, 8582, 8581, 5775, 8582] avg=8192.0000 max=8583 max_rank=1 min=5775 min_rank=6 imbalance_ratio=1.047729 max_load_reduction=22205 max_load_reduction_pct=0.721223 moved_to_replicas=71568 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[3351, 10549, 1, 923, 27535, 602, 83, 22492] avg=8192.0000 max=27535 max_rank=4 min=1 min_rank=2 imbalance_ratio=3.361206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[3351, 10549, 1, 923, 27535, 602, 83, 22492] avg=8192.0000 max=27535 max_rank=4 min=1 min_rank=2 imbalance_ratio=3.361206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[239, 50, 20848, 6128, 30788, 3199, 445, 3839] avg=8192.0000 max=30788 max_rank=4 min=50 min_rank=1 imbalance_ratio=3.758301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[8582, 8583, 8583, 8268, 8582, 8581, 5775, 8582] avg=8192.0000 max=8583 max_rank=1 min=5775 min_rank=6 imbalance_ratio=1.047729 max_load_reduction=22205 max_load_reduction_pct=0.721223 moved_to_replicas=71568 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[3351, 10549, 11547, 923, 11547, 11547, 4525, 11547] avg=8192.0000 max=11547 max_rank=2 min=923 min_rank=3 imbalance_ratio=1.409546 max_load_reduction=15988 max_load_reduction_pct=0.580643 moved_to_replicas=62239 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[3351, 10549, 1, 923, 27535, 602, 83, 22492] avg=8192.0000 max=27535 max_rank=4 min=1 min_rank=2 imbalance_ratio=3.361206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[239, 50, 20848, 6128, 30788, 3199, 445, 3839] avg=8192.0000 max=30788 max_rank=4 min=50 min_rank=1 imbalance_ratio=3.758301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[3351, 10549, 11547, 923, 11547, 11547, 4525, 11547] avg=8192.0000 max=11547 max_rank=2 min=923 min_rank=3 imbalance_ratio=1.409546 max_load_reduction=15988 max_load_reduction_pct=0.580643 moved_to_replicas=62239 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[3351, 10549, 11547, 923, 11547, 11547, 4525, 11547] avg=8192.0000 max=11547 max_rank=2 min=923 min_rank=3 imbalance_ratio=1.409546 max_load_reduction=15988 max_load_reduction_pct=0.580643 moved_to_replicas=62239 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[8582, 8583, 8583, 8268, 8582, 8581, 5775, 8582] avg=8192.0000 max=8583 max_rank=1 min=5775 min_rank=6 imbalance_ratio=1.047729 max_load_reduction=22205 max_load_reduction_pct=0.721223 moved_to_replicas=71568 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[8582, 8583, 8583, 8268, 8582, 8581, 5775, 8582] avg=8192.0000 max=8583 max_rank=1 min=5775 min_rank=6 imbalance_ratio=1.047729 max_load_reduction=22205 max_load_reduction_pct=0.721223 moved_to_replicas=71568 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[3351, 10549, 1, 923, 27535, 602, 83, 22492] avg=8192.0000 max=27535 max_rank=4 min=1 min_rank=2 imbalance_ratio=3.361206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[3351, 10549, 11547, 923, 11547, 11547, 4525, 11547] avg=8192.0000 max=11547 max_rank=2 min=923 min_rank=3 imbalance_ratio=1.409546 max_load_reduction=15988 max_load_reduction_pct=0.580643 moved_to_replicas=62239 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[279, 2569, 756, 15983, 19481, 48, 451, 25969] avg=8192.0000 max=25969 max_rank=7 min=48 min_rank=5 imbalance_ratio=3.170044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[3351, 10549, 1, 923, 27535, 602, 83, 22492] avg=8192.0000 max=27535 max_rank=4 min=1 min_rank=2 imbalance_ratio=3.361206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[279, 2569, 756, 15983, 19481, 48, 451, 25969] avg=8192.0000 max=25969 max_rank=7 min=48 min_rank=5 imbalance_ratio=3.170044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[279, 2569, 756, 15983, 19481, 48, 451, 25969] avg=8192.0000 max=25969 max_rank=7 min=48 min_rank=5 imbalance_ratio=3.170044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[3351, 10549, 11547, 923, 11547, 11547, 4525, 11547] avg=8192.0000 max=11547 max_rank=2 min=923 min_rank=3 imbalance_ratio=1.409546 max_load_reduction=15988 max_load_reduction_pct=0.580643 moved_to_replicas=62239 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[6364, 2569, 6772, 9967, 9966, 9966, 9966, 9966] avg=8192.0000 max=9967 max_rank=3 min=2569 min_rank=1 imbalance_ratio=1.216675 max_load_reduction=16002 max_load_reduction_pct=0.616196 moved_to_replicas=61234 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[3351, 10549, 11547, 923, 11547, 11547, 4525, 11547] avg=8192.0000 max=11547 max_rank=2 min=923 min_rank=3 imbalance_ratio=1.409546 max_load_reduction=15988 max_load_reduction_pct=0.580643 moved_to_replicas=62239 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[3351, 10549, 1, 923, 27535, 602, 83, 22492] avg=8192.0000 max=27535 max_rank=4 min=1 min_rank=2 imbalance_ratio=3.361206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[3351, 10549, 1, 923, 27535, 602, 83, 22492] avg=8192.0000 max=27535 max_rank=4 min=1 min_rank=2 imbalance_ratio=3.361206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[279, 2569, 756, 15983, 19481, 48, 451, 25969] avg=8192.0000 max=25969 max_rank=7 min=48 min_rank=5 imbalance_ratio=3.170044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[6364, 2569, 6772, 9967, 9966, 9966, 9966, 9966] avg=8192.0000 max=9967 max_rank=3 min=2569 min_rank=1 imbalance_ratio=1.216675 max_load_reduction=16002 max_load_reduction_pct=0.616196 moved_to_replicas=61234 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[6364, 2569, 6772, 9967, 9966, 9966, 9966, 9966] avg=8192.0000 max=9967 max_rank=3 min=2569 min_rank=1 imbalance_ratio=1.216675 max_load_reduction=16002 max_load_reduction_pct=0.616196 moved_to_replicas=61234 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[3351, 10549, 11547, 923, 11547, 11547, 4525, 11547] avg=8192.0000 max=11547 max_rank=2 min=923 min_rank=3 imbalance_ratio=1.409546 max_load_reduction=15988 max_load_reduction_pct=0.580643 moved_to_replicas=62239 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[3351, 10549, 11547, 923, 11547, 11547, 4525, 11547] avg=8192.0000 max=11547 max_rank=2 min=923 min_rank=3 imbalance_ratio=1.409546 max_load_reduction=15988 max_load_reduction_pct=0.580643 moved_to_replicas=62239 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[279, 2569, 756, 15983, 19481, 48, 451, 25969] avg=8192.0000 max=25969 max_rank=7 min=48 min_rank=5 imbalance_ratio=3.170044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[279, 2569, 756, 15983, 19481, 48, 451, 25969] avg=8192.0000 max=25969 max_rank=7 min=48 min_rank=5 imbalance_ratio=3.170044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[6364, 2569, 6772, 9967, 9966, 9966, 9966, 9966] avg=8192.0000 max=9967 max_rank=3 min=2569 min_rank=1 imbalance_ratio=1.216675 max_load_reduction=16002 max_load_reduction_pct=0.616196 moved_to_replicas=61234 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[3, 86, 1173, 16412, 19753, 25638, 320, 2151] avg=8192.0000 max=25638 max_rank=5 min=3 min_rank=0 imbalance_ratio=3.129639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[3, 86, 1173, 16412, 19753, 25638, 320, 2151] avg=8192.0000 max=25638 max_rank=5 min=3 min_rank=0 imbalance_ratio=3.129639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[3, 86, 1173, 16412, 19753, 25638, 320, 2151] avg=8192.0000 max=25638 max_rank=5 min=3 min_rank=0 imbalance_ratio=3.129639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[6364, 2569, 6772, 9967, 9966, 9966, 9966, 9966] avg=8192.0000 max=9967 max_rank=3 min=2569 min_rank=1 imbalance_ratio=1.216675 max_load_reduction=16002 max_load_reduction_pct=0.616196 moved_to_replicas=61234 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[279, 2569, 756, 15983, 19481, 48, 451, 25969] avg=8192.0000 max=25969 max_rank=7 min=48 min_rank=5 imbalance_ratio=3.170044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[279, 2569, 756, 15983, 19481, 48, 451, 25969] avg=8192.0000 max=25969 max_rank=7 min=48 min_rank=5 imbalance_ratio=3.170044 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[6364, 2569, 6772, 9967, 9966, 9966, 9966, 9966] avg=8192.0000 max=9967 max_rank=3 min=2569 min_rank=1 imbalance_ratio=1.216675 max_load_reduction=16002 max_load_reduction_pct=0.616196 moved_to_replicas=61234 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[10036, 5655, 7547, 10038, 10037, 10036, 10036, 2151] avg=8192.0000 max=10038 max_rank=3 min=2151 min_rank=7 imbalance_ratio=1.225342 max_load_reduction=15600 max_load_reduction_pct=0.608472 moved_to_replicas=69004 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[10036, 5655, 7547, 10038, 10037, 10036, 10036, 2151] avg=8192.0000 max=10038 max_rank=3 min=2151 min_rank=7 imbalance_ratio=1.225342 max_load_reduction=15600 max_load_reduction_pct=0.608472 moved_to_replicas=69004 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[10036, 5655, 7547, 10038, 10037, 10036, 10036, 2151] avg=8192.0000 max=10038 max_rank=3 min=2151 min_rank=7 imbalance_ratio=1.225342 max_load_reduction=15600 max_load_reduction_pct=0.608472 moved_to_replicas=69004 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[3, 86, 1173, 16412, 19753, 25638, 320, 2151] avg=8192.0000 max=25638 max_rank=5 min=3 min_rank=0 imbalance_ratio=3.129639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[6364, 2569, 6772, 9967, 9966, 9966, 9966, 9966] avg=8192.0000 max=9967 max_rank=3 min=2569 min_rank=1 imbalance_ratio=1.216675 max_load_reduction=16002 max_load_reduction_pct=0.616196 moved_to_replicas=61234 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[6364, 2569, 6772, 9967, 9966, 9966, 9966, 9966] avg=8192.0000 max=9967 max_rank=3 min=2569 min_rank=1 imbalance_ratio=1.216675 max_load_reduction=16002 max_load_reduction_pct=0.616196 moved_to_replicas=61234 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[3, 86, 1173, 16412, 19753, 25638, 320, 2151] avg=8192.0000 max=25638 max_rank=5 min=3 min_rank=0 imbalance_ratio=3.129639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[3, 86, 1173, 16412, 19753, 25638, 320, 2151] avg=8192.0000 max=25638 max_rank=5 min=3 min_rank=0 imbalance_ratio=3.129639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[10036, 5655, 7547, 10038, 10037, 10036, 10036, 2151] avg=8192.0000 max=10038 max_rank=3 min=2151 min_rank=7 imbalance_ratio=1.225342 max_load_reduction=15600 max_load_reduction_pct=0.608472 moved_to_replicas=69004 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[14846, 6125, 31563, 456, 3429, 641, 8192, 284] avg=8192.0000 max=31563 max_rank=2 min=284 min_rank=7 imbalance_ratio=3.852905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[14846, 6125, 31563, 456, 3429, 641, 8192, 284] avg=8192.0000 max=31563 max_rank=2 min=284 min_rank=7 imbalance_ratio=3.852905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[14846, 6125, 31563, 456, 3429, 641, 8192, 284] avg=8192.0000 max=31563 max_rank=2 min=284 min_rank=7 imbalance_ratio=3.852905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[10036, 5655, 7547, 10038, 10037, 10036, 10036, 2151] avg=8192.0000 max=10038 max_rank=3 min=2151 min_rank=7 imbalance_ratio=1.225342 max_load_reduction=15600 max_load_reduction_pct=0.608472 moved_to_replicas=69004 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[3, 86, 1173, 16412, 19753, 25638, 320, 2151] avg=8192.0000 max=25638 max_rank=5 min=3 min_rank=0 imbalance_ratio=3.129639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[3, 86, 1173, 16412, 19753, 25638, 320, 2151] avg=8192.0000 max=25638 max_rank=5 min=3 min_rank=0 imbalance_ratio=3.129639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[10036, 5655, 7547, 10038, 10037, 10036, 10036, 2151] avg=8192.0000 max=10038 max_rank=3 min=2151 min_rank=7 imbalance_ratio=1.225342 max_load_reduction=15600 max_load_reduction_pct=0.608472 moved_to_replicas=69004 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[8273, 7852, 8274, 8273, 8275, 8123, 8192, 8274] avg=8192.0000 max=8275 max_rank=4 min=7852 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=23288 max_load_reduction_pct=0.737826 moved_to_replicas=61819 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[8273, 7852, 8274, 8273, 8275, 8123, 8192, 8274] avg=8192.0000 max=8275 max_rank=4 min=7852 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=23288 max_load_reduction_pct=0.737826 moved_to_replicas=61819 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[8273, 7852, 8274, 8273, 8275, 8123, 8192, 8274] avg=8192.0000 max=8275 max_rank=4 min=7852 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=23288 max_load_reduction_pct=0.737826 moved_to_replicas=61819 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[14846, 6125, 31563, 456, 3429, 641, 8192, 284] avg=8192.0000 max=31563 max_rank=2 min=284 min_rank=7 imbalance_ratio=3.852905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[10036, 5655, 7547, 10038, 10037, 10036, 10036, 2151] avg=8192.0000 max=10038 max_rank=3 min=2151 min_rank=7 imbalance_ratio=1.225342 max_load_reduction=15600 max_load_reduction_pct=0.608472 moved_to_replicas=69004 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[10036, 5655, 7547, 10038, 10037, 10036, 10036, 2151] avg=8192.0000 max=10038 max_rank=3 min=2151 min_rank=7 imbalance_ratio=1.225342 max_load_reduction=15600 max_load_reduction_pct=0.608472 moved_to_replicas=69004 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[14846, 6125, 31563, 456, 3429, 641, 8192, 284] avg=8192.0000 max=31563 max_rank=2 min=284 min_rank=7 imbalance_ratio=3.852905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[14846, 6125, 31563, 456, 3429, 641, 8192, 284] avg=8192.0000 max=31563 max_rank=2 min=284 min_rank=7 imbalance_ratio=3.852905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[8273, 7852, 8274, 8273, 8275, 8123, 8192, 8274] avg=8192.0000 max=8275 max_rank=4 min=7852 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=23288 max_load_reduction_pct=0.737826 moved_to_replicas=61819 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[1090, 27237, 156, 4183, 35, 1125, 28878, 2832] avg=8192.0000 max=28878 max_rank=6 min=35 min_rank=4 imbalance_ratio=3.525146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[1090, 27237, 156, 4183, 35, 1125, 28878, 2832] avg=8192.0000 max=28878 max_rank=6 min=35 min_rank=4 imbalance_ratio=3.525146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[1090, 27237, 156, 4183, 35, 1125, 28878, 2832] avg=8192.0000 max=28878 max_rank=6 min=35 min_rank=4 imbalance_ratio=3.525146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[8273, 7852, 8274, 8273, 8275, 8123, 8192, 8274] avg=8192.0000 max=8275 max_rank=4 min=7852 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=23288 max_load_reduction_pct=0.737826 moved_to_replicas=61819 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[14846, 6125, 31563, 456, 3429, 641, 8192, 284] avg=8192.0000 max=31563 max_rank=2 min=284 min_rank=7 imbalance_ratio=3.852905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[14846, 6125, 31563, 456, 3429, 641, 8192, 284] avg=8192.0000 max=31563 max_rank=2 min=284 min_rank=7 imbalance_ratio=3.852905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[8273, 7852, 8274, 8273, 8275, 8123, 8192, 8274] avg=8192.0000 max=8275 max_rank=4 min=7852 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=23288 max_load_reduction_pct=0.737826 moved_to_replicas=61819 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[3603, 8853, 8853, 8821, 8851, 8851, 8852, 8852] avg=8192.0000 max=8853 max_rank=1 min=3603 min_rank=0 imbalance_ratio=1.080688 max_load_reduction=20025 max_load_reduction_pct=0.693434 moved_to_replicas=60128 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[3603, 8853, 8853, 8821, 8851, 8851, 8852, 8852] avg=8192.0000 max=8853 max_rank=1 min=3603 min_rank=0 imbalance_ratio=1.080688 max_load_reduction=20025 max_load_reduction_pct=0.693434 moved_to_replicas=60128 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[3603, 8853, 8853, 8821, 8851, 8851, 8852, 8852] avg=8192.0000 max=8853 max_rank=1 min=3603 min_rank=0 imbalance_ratio=1.080688 max_load_reduction=20025 max_load_reduction_pct=0.693434 moved_to_replicas=60128 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[1090, 27237, 156, 4183, 35, 1125, 28878, 2832] avg=8192.0000 max=28878 max_rank=6 min=35 min_rank=4 imbalance_ratio=3.525146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[8273, 7852, 8274, 8273, 8275, 8123, 8192, 8274] avg=8192.0000 max=8275 max_rank=4 min=7852 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=23288 max_load_reduction_pct=0.737826 moved_to_replicas=61819 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[8273, 7852, 8274, 8273, 8275, 8123, 8192, 8274] avg=8192.0000 max=8275 max_rank=4 min=7852 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=23288 max_load_reduction_pct=0.737826 moved_to_replicas=61819 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[1090, 27237, 156, 4183, 35, 1125, 28878, 2832] avg=8192.0000 max=28878 max_rank=6 min=35 min_rank=4 imbalance_ratio=3.525146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[8987, 8216, 7291, 26198, 293, 13594, 65, 892] avg=8192.0000 max=26198 max_rank=3 min=65 min_rank=6 imbalance_ratio=3.197998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[3603, 8853, 8853, 8821, 8851, 8851, 8852, 8852] avg=8192.0000 max=8853 max_rank=1 min=3603 min_rank=0 imbalance_ratio=1.080688 max_load_reduction=20025 max_load_reduction_pct=0.693434 moved_to_replicas=60128 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[1090, 27237, 156, 4183, 35, 1125, 28878, 2832] avg=8192.0000 max=28878 max_rank=6 min=35 min_rank=4 imbalance_ratio=3.525146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[8987, 8216, 7291, 26198, 293, 13594, 65, 892] avg=8192.0000 max=26198 max_rank=3 min=65 min_rank=6 imbalance_ratio=3.197998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[8987, 8216, 7291, 26198, 293, 13594, 65, 892] avg=8192.0000 max=26198 max_rank=3 min=65 min_rank=6 imbalance_ratio=3.197998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[3603, 8853, 8853, 8821, 8851, 8851, 8852, 8852] avg=8192.0000 max=8853 max_rank=1 min=3603 min_rank=0 imbalance_ratio=1.080688 max_load_reduction=20025 max_load_reduction_pct=0.693434 moved_to_replicas=60128 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[1090, 27237, 156, 4183, 35, 1125, 28878, 2832] avg=8192.0000 max=28878 max_rank=6 min=35 min_rank=4 imbalance_ratio=3.525146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[1090, 27237, 156, 4183, 35, 1125, 28878, 2832] avg=8192.0000 max=28878 max_rank=6 min=35 min_rank=4 imbalance_ratio=3.525146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[8987, 8216, 7291, 8986, 8582, 8987, 8988, 5499] avg=8192.0000 max=8988 max_rank=6 min=5499 min_rank=7 imbalance_ratio=1.097168 max_load_reduction=17210 max_load_reduction_pct=0.656920 moved_to_replicas=51779 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[3603, 8853, 8853, 8821, 8851, 8851, 8852, 8852] avg=8192.0000 max=8853 max_rank=1 min=3603 min_rank=0 imbalance_ratio=1.080688 max_load_reduction=20025 max_load_reduction_pct=0.693434 moved_to_replicas=60128 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[8987, 8216, 7291, 8986, 8582, 8987, 8988, 5499] avg=8192.0000 max=8988 max_rank=6 min=5499 min_rank=7 imbalance_ratio=1.097168 max_load_reduction=17210 max_load_reduction_pct=0.656920 moved_to_replicas=51779 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[8987, 8216, 7291, 8986, 8582, 8987, 8988, 5499] avg=8192.0000 max=8988 max_rank=6 min=5499 min_rank=7 imbalance_ratio=1.097168 max_load_reduction=17210 max_load_reduction_pct=0.656920 moved_to_replicas=51779 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[8987, 8216, 7291, 26198, 293, 13594, 65, 892] avg=8192.0000 max=26198 max_rank=3 min=65 min_rank=6 imbalance_ratio=3.197998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[3603, 8853, 8853, 8821, 8851, 8851, 8852, 8852] avg=8192.0000 max=8853 max_rank=1 min=3603 min_rank=0 imbalance_ratio=1.080688 max_load_reduction=20025 max_load_reduction_pct=0.693434 moved_to_replicas=60128 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[3603, 8853, 8853, 8821, 8851, 8851, 8852, 8852] avg=8192.0000 max=8853 max_rank=1 min=3603 min_rank=0 imbalance_ratio=1.080688 max_load_reduction=20025 max_load_reduction_pct=0.693434 moved_to_replicas=60128 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[11350, 1756, 10288, 22550, 12792, 132, 53, 6615] avg=8192.0000 max=22550 max_rank=3 min=53 min_rank=6 imbalance_ratio=2.752686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[8987, 8216, 7291, 26198, 293, 13594, 65, 892] avg=8192.0000 max=26198 max_rank=3 min=65 min_rank=6 imbalance_ratio=3.197998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[8987, 8216, 7291, 8986, 8582, 8987, 8988, 5499] avg=8192.0000 max=8988 max_rank=6 min=5499 min_rank=7 imbalance_ratio=1.097168 max_load_reduction=17210 max_load_reduction_pct=0.656920 moved_to_replicas=51779 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[11350, 1756, 10288, 22550, 12792, 132, 53, 6615] avg=8192.0000 max=22550 max_rank=3 min=53 min_rank=6 imbalance_ratio=2.752686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[11350, 1756, 10288, 22550, 12792, 132, 53, 6615] avg=8192.0000 max=22550 max_rank=3 min=53 min_rank=6 imbalance_ratio=2.752686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[8987, 8216, 7291, 26198, 293, 13594, 65, 892] avg=8192.0000 max=26198 max_rank=3 min=65 min_rank=6 imbalance_ratio=3.197998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[8987, 8216, 7291, 26198, 293, 13594, 65, 892] avg=8192.0000 max=26198 max_rank=3 min=65 min_rank=6 imbalance_ratio=3.197998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[8987, 8216, 7291, 26198, 293, 13594, 65, 892] avg=8192.0000 max=26198 max_rank=3 min=65 min_rank=6 imbalance_ratio=3.197998 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[11350, 1756, 10288, 11350, 11350, 1574, 11253, 6615] avg=8192.0000 max=11350 max_rank=0 min=1574 min_rank=5 imbalance_ratio=1.385498 max_load_reduction=11200 max_load_reduction_pct=0.496674 moved_to_replicas=40798 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[8987, 8216, 7291, 8986, 8582, 8987, 8988, 5499] avg=8192.0000 max=8988 max_rank=6 min=5499 min_rank=7 imbalance_ratio=1.097168 max_load_reduction=17210 max_load_reduction_pct=0.656920 moved_to_replicas=51779 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[11350, 1756, 10288, 11350, 11350, 1574, 11253, 6615] avg=8192.0000 max=11350 max_rank=0 min=1574 min_rank=5 imbalance_ratio=1.385498 max_load_reduction=11200 max_load_reduction_pct=0.496674 moved_to_replicas=40798 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[11350, 1756, 10288, 11350, 11350, 1574, 11253, 6615] avg=8192.0000 max=11350 max_rank=0 min=1574 min_rank=5 imbalance_ratio=1.385498 max_load_reduction=11200 max_load_reduction_pct=0.496674 moved_to_replicas=40798 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[11350, 1756, 10288, 22550, 12792, 132, 53, 6615] avg=8192.0000 max=22550 max_rank=3 min=53 min_rank=6 imbalance_ratio=2.752686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[8987, 8216, 7291, 8986, 8582, 8987, 8988, 5499] avg=8192.0000 max=8988 max_rank=6 min=5499 min_rank=7 imbalance_ratio=1.097168 max_load_reduction=17210 max_load_reduction_pct=0.656920 moved_to_replicas=51779 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[8987, 8216, 7291, 8986, 8582, 8987, 8988, 5499] avg=8192.0000 max=8988 max_rank=6 min=5499 min_rank=7 imbalance_ratio=1.097168 max_load_reduction=17210 max_load_reduction_pct=0.656920 moved_to_replicas=51779 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[8987, 8216, 7291, 8986, 8582, 8987, 8988, 5499] avg=8192.0000 max=8988 max_rank=6 min=5499 min_rank=7 imbalance_ratio=1.097168 max_load_reduction=17210 max_load_reduction_pct=0.656920 moved_to_replicas=51779 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[1513, 82, 340, 25469, 130, 32318, 4881, 803] avg=8192.0000 max=32318 max_rank=5 min=82 min_rank=1 imbalance_ratio=3.945068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[1513, 82, 340, 25469, 130, 32318, 4881, 803] avg=8192.0000 max=32318 max_rank=5 min=82 min_rank=1 imbalance_ratio=3.945068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[11350, 1756, 10288, 11350, 11350, 1574, 11253, 6615] avg=8192.0000 max=11350 max_rank=0 min=1574 min_rank=5 imbalance_ratio=1.385498 max_load_reduction=11200 max_load_reduction_pct=0.496674 moved_to_replicas=40798 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[11350, 1756, 10288, 22550, 12792, 132, 53, 6615] avg=8192.0000 max=22550 max_rank=3 min=53 min_rank=6 imbalance_ratio=2.752686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[1513, 82, 340, 25469, 130, 32318, 4881, 803] avg=8192.0000 max=32318 max_rank=5 min=82 min_rank=1 imbalance_ratio=3.945068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[11350, 1756, 10288, 22550, 12792, 132, 53, 6615] avg=8192.0000 max=22550 max_rank=3 min=53 min_rank=6 imbalance_ratio=2.752686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[11350, 1756, 10288, 22550, 12792, 132, 53, 6615] avg=8192.0000 max=22550 max_rank=3 min=53 min_rank=6 imbalance_ratio=2.752686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[11350, 1756, 10288, 22550, 12792, 132, 53, 6615] avg=8192.0000 max=22550 max_rank=3 min=53 min_rank=6 imbalance_ratio=2.752686 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[8274, 8274, 8048, 8274, 8274, 8274, 7844, 8274] avg=8192.0000 max=8274 max_rank=0 min=7844 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24044 max_load_reduction_pct=0.743982 moved_to_replicas=83585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[8274, 8274, 8048, 8274, 8274, 8274, 7844, 8274] avg=8192.0000 max=8274 max_rank=0 min=7844 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24044 max_load_reduction_pct=0.743982 moved_to_replicas=83585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[11350, 1756, 10288, 11350, 11350, 1574, 11253, 6615] avg=8192.0000 max=11350 max_rank=0 min=1574 min_rank=5 imbalance_ratio=1.385498 max_load_reduction=11200 max_load_reduction_pct=0.496674 moved_to_replicas=40798 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[8274, 8274, 8048, 8274, 8274, 8274, 7844, 8274] avg=8192.0000 max=8274 max_rank=0 min=7844 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24044 max_load_reduction_pct=0.743982 moved_to_replicas=83585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[11350, 1756, 10288, 11350, 11350, 1574, 11253, 6615] avg=8192.0000 max=11350 max_rank=0 min=1574 min_rank=5 imbalance_ratio=1.385498 max_load_reduction=11200 max_load_reduction_pct=0.496674 moved_to_replicas=40798 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[1513, 82, 340, 25469, 130, 32318, 4881, 803] avg=8192.0000 max=32318 max_rank=5 min=82 min_rank=1 imbalance_ratio=3.945068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[11350, 1756, 10288, 11350, 11350, 1574, 11253, 6615] avg=8192.0000 max=11350 max_rank=0 min=1574 min_rank=5 imbalance_ratio=1.385498 max_load_reduction=11200 max_load_reduction_pct=0.496674 moved_to_replicas=40798 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[11350, 1756, 10288, 11350, 11350, 1574, 11253, 6615] avg=8192.0000 max=11350 max_rank=0 min=1574 min_rank=5 imbalance_ratio=1.385498 max_load_reduction=11200 max_load_reduction_pct=0.496674 moved_to_replicas=40798 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[1513, 82, 340, 25469, 130, 32318, 4881, 803] avg=8192.0000 max=32318 max_rank=5 min=82 min_rank=1 imbalance_ratio=3.945068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[8274, 8274, 8048, 8274, 8274, 8274, 7844, 8274] avg=8192.0000 max=8274 max_rank=0 min=7844 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24044 max_load_reduction_pct=0.743982 moved_to_replicas=83585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[1513, 82, 340, 25469, 130, 32318, 4881, 803] avg=8192.0000 max=32318 max_rank=5 min=82 min_rank=1 imbalance_ratio=3.945068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[1513, 82, 340, 25469, 130, 32318, 4881, 803] avg=8192.0000 max=32318 max_rank=5 min=82 min_rank=1 imbalance_ratio=3.945068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[1513, 82, 340, 25469, 130, 32318, 4881, 803] avg=8192.0000 max=32318 max_rank=5 min=82 min_rank=1 imbalance_ratio=3.945068 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[8274, 8274, 8048, 8274, 8274, 8274, 7844, 8274] avg=8192.0000 max=8274 max_rank=0 min=7844 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24044 max_load_reduction_pct=0.743982 moved_to_replicas=83585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[8274, 8274, 8048, 8274, 8274, 8274, 7844, 8274] avg=8192.0000 max=8274 max_rank=0 min=7844 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24044 max_load_reduction_pct=0.743982 moved_to_replicas=83585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[8274, 8274, 8048, 8274, 8274, 8274, 7844, 8274] avg=8192.0000 max=8274 max_rank=0 min=7844 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24044 max_load_reduction_pct=0.743982 moved_to_replicas=83585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[8274, 8274, 8048, 8274, 8274, 8274, 7844, 8274] avg=8192.0000 max=8274 max_rank=0 min=7844 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24044 max_load_reduction_pct=0.743982 moved_to_replicas=83585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=32 total_expected_recv=246788 total_dropped_tokens=1722 drop_rate_pct=0.697765 records_with_drop_pct=6.250000 peak_rank_overflow=1307 total_slack_slots=79488 peak_demand=11547 required_capacity=1.409546 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=32 total_expected_recv=260336 total_dropped_tokens=2432 drop_rate_pct=0.934177 records_with_drop_pct=6.250000 peak_rank_overflow=2123 total_slack_slots=66432 peak_demand=12363 required_capacity=1.509155 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=32 total_expected_recv=278254 total_dropped_tokens=4953 drop_rate_pct=1.780028 records_with_drop_pct=12.500000 peak_rank_overflow=2121 total_slack_slots=51712 peak_demand=12361 required_capacity=1.508911 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=32 total_expected_recv=243892 total_dropped_tokens=1524 drop_rate_pct=0.624867 records_with_drop_pct=6.250000 peak_rank_overflow=1110 total_slack_slots=82176 peak_demand=11350 required_capacity=1.385498 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=32 total_expected_recv=268506 total_dropped_tokens=3136 drop_rate_pct=1.167944 records_with_drop_pct=6.250000 peak_rank_overflow=2123 total_slack_slots=59264 peak_demand=12363 required_capacity=1.509155 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=32 total_expected_recv=252060 total_dropped_tokens=1674 drop_rate_pct=0.664128 records_with_drop_pct=6.250000 peak_rank_overflow=1307 total_slack_slots=73984 peak_demand=11547 required_capacity=1.409546 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=32 total_expected_recv=272615 total_dropped_tokens=3130 drop_rate_pct=1.148139 records_with_drop_pct=9.375000 peak_rank_overflow=1604 total_slack_slots=55296 peak_demand=11844 required_capacity=1.445801 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=32 total_expected_recv=274701 total_dropped_tokens=1355 drop_rate_pct=0.493264 records_with_drop_pct=6.250000 peak_rank_overflow=1307 total_slack_slots=51200 peak_demand=11547 required_capacity=1.409546 + [2026-08-30 05:07:58] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 860.5 | throughput per GPU (TFLOP/s/GPU): 387.4 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 9.676978E+00 | loss scale: 1.0 | grad norm: 41.863 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 10240 | min safe capacity: 1.2500 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=10240 expected_recv=16949 expected_overflow=0 exact_recv_per_rank=[8580, 8580, 8365, 7528, 8580, 6743, 8580, 8580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=10240 expected_recv=17165 expected_overflow=0 exact_recv_per_rank=[8580, 8580, 8365, 7528, 8580, 6743, 8580, 8580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=10240 expected_recv=17164 expected_overflow=0 exact_recv_per_rank=[8580, 8580, 8365, 7528, 8580, 6743, 8580, 8580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=10240 expected_recv=13244 expected_overflow=0 exact_recv_per_rank=[8580, 8580, 8365, 7528, 8580, 6743, 8580, 8580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=10240 expected_recv=15059 expected_overflow=0 exact_recv_per_rank=[8580, 8580, 8365, 7528, 8580, 6743, 8580, 8580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=10240 expected_recv=17165 expected_overflow=0 exact_recv_per_rank=[8580, 8580, 8365, 7528, 8580, 6743, 8580, 8580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=10240 expected_recv=17163 expected_overflow=0 exact_recv_per_rank=[8580, 8580, 8365, 7528, 8580, 6743, 8580, 8580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=10240 expected_recv=16949 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=10240 expected_recv=17165 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=10240 expected_recv=17163 expected_overflow=0 exact_recv_per_rank=[8580, 8580, 8365, 7528, 8580, 6743, 8580, 8580] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=10240 expected_recv=17164 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=10240 expected_recv=13244 actual_valid_recv=13568 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=10240 expected_recv=15059 actual_valid_recv=15360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=10240 expected_recv=17165 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17152 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=10240 expected_recv=17163 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=13568 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=15360 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17536 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=10240 expected_recv=17163 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=10240 expected_recv=14981 expected_overflow=0 exact_recv_per_rank=[8596, 8595, 6386, 8596, 7983, 8187, 8596, 8597] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=10240 expected_recv=17189 expected_overflow=0 exact_recv_per_rank=[8596, 8595, 6386, 8596, 7983, 8187, 8596, 8597] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=10240 expected_recv=17190 expected_overflow=0 exact_recv_per_rank=[8596, 8595, 6386, 8596, 7983, 8187, 8596, 8597] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=10240 expected_recv=17190 expected_overflow=0 exact_recv_per_rank=[8596, 8595, 6386, 8596, 7983, 8187, 8596, 8597] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=10240 expected_recv=16524 expected_overflow=0 exact_recv_per_rank=[8596, 8595, 6386, 8596, 7983, 8187, 8596, 8597] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=10240 expected_recv=14918 expected_overflow=0 exact_recv_per_rank=[8596, 8595, 6386, 8596, 7983, 8187, 8596, 8597] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=10240 expected_recv=14981 actual_valid_recv=15360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=10240 expected_recv=17189 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=10240 expected_recv=17190 expected_overflow=0 exact_recv_per_rank=[8596, 8595, 6386, 8596, 7983, 8187, 8596, 8597] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=10240 expected_recv=17190 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=10240 expected_recv=17190 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=10240 expected_recv=16524 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=10240 expected_recv=15890 expected_overflow=0 exact_recv_per_rank=[8596, 8595, 6386, 8596, 7983, 8187, 8596, 8597] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=15360 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=10240 expected_recv=14918 actual_valid_recv=15104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=17536 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=17664 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=10240 expected_recv=17190 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=15104 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=10240 expected_recv=15890 actual_valid_recv=16000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16000 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=10240 expected_recv=15670 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 7903, 7988, 8274, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 7903, 7988, 8274, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=10240 expected_recv=16113 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 7903, 7988, 8274, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 7903, 7988, 8274, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 7903, 7988, 8274, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 7903, 7988, 8274, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=10240 expected_recv=15670 actual_valid_recv=16000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 7903, 7988, 8274, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 7903, 7988, 8274, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=10240 expected_recv=16113 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16000 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=10240 expected_recv=17179 expected_overflow=0 exact_recv_per_rank=[8590, 8590, 8589, 5855, 8590, 8590, 8142, 8590] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=10240 expected_recv=11737 expected_overflow=0 exact_recv_per_rank=[8590, 8590, 8589, 5855, 8590, 8590, 8142, 8590] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=10240 expected_recv=17180 expected_overflow=0 exact_recv_per_rank=[8590, 8590, 8589, 5855, 8590, 8590, 8142, 8590] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=10240 expected_recv=16256 expected_overflow=0 exact_recv_per_rank=[8590, 8590, 8589, 5855, 8590, 8590, 8142, 8590] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=10240 expected_recv=17180 expected_overflow=0 exact_recv_per_rank=[8590, 8590, 8589, 5855, 8590, 8590, 8142, 8590] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=10240 expected_recv=17179 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=10240 expected_recv=11737 actual_valid_recv=12032 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=10240 expected_recv=17180 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=10240 expected_recv=17179 expected_overflow=0 exact_recv_per_rank=[8590, 8590, 8589, 5855, 8590, 8590, 8142, 8590] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=10240 expected_recv=17181 expected_overflow=0 exact_recv_per_rank=[8590, 8590, 8589, 5855, 8590, 8590, 8142, 8590] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=10240 expected_recv=17180 expected_overflow=0 exact_recv_per_rank=[8590, 8590, 8589, 5855, 8590, 8590, 8142, 8590] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=10240 expected_recv=16256 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=12032 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=10240 expected_recv=17180 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=10240 expected_recv=17179 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=10240 expected_recv=17181 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=10240 expected_recv=17180 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17536 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17536 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=10240 expected_recv=16800 expected_overflow=652 exact_recv_per_rank=[6355, 8634, 8927, 5908, 8927, 8929, 8928, 8928] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=10240 expected_recv=8995 expected_overflow=0 exact_recv_per_rank=[6355, 8634, 8927, 5908, 8927, 8929, 8928, 8928] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=10240 expected_recv=19526 expected_overflow=652 exact_recv_per_rank=[6355, 8634, 8927, 5908, 8927, 8929, 8928, 8928] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=10240 expected_recv=16800 actual_valid_recv=16384 observed_loss=652 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=10240 expected_recv=8995 actual_valid_recv=9216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=10240 expected_recv=19526 actual_valid_recv=18944 observed_loss=652 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=10240 expected_recv=19658 expected_overflow=490 exact_recv_per_rank=[6355, 8634, 8927, 5908, 8927, 8929, 8928, 8928] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=9216 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=10240 expected_recv=19821 expected_overflow=653 exact_recv_per_rank=[6355, 8634, 8927, 5908, 8927, 8929, 8928, 8928] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=10240 expected_recv=6685 expected_overflow=0 exact_recv_per_rank=[6355, 8634, 8927, 5908, 8927, 8929, 8928, 8928] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=18944 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=10240 expected_recv=19822 expected_overflow=653 exact_recv_per_rank=[6355, 8634, 8927, 5908, 8927, 8929, 8928, 8928] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=10240 expected_recv=19765 expected_overflow=598 exact_recv_per_rank=[6355, 8634, 8927, 5908, 8927, 8929, 8928, 8928] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=10240 expected_recv=19658 actual_valid_recv=19328 observed_loss=490 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=10240 expected_recv=19821 actual_valid_recv=19200 observed_loss=653 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=10240 expected_recv=6685 actual_valid_recv=6912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=19328 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=10240 expected_recv=19822 actual_valid_recv=19200 observed_loss=653 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=10240 expected_recv=19765 actual_valid_recv=19328 observed_loss=598 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=19200 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=6912 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=19200 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=19328 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8103, 8275, 7789, 8273, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=10240 expected_recv=16377 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8103, 8275, 7789, 8273, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8103, 8275, 7789, 8273, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=10240 expected_recv=16377 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=10240 expected_recv=16402 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8103, 8275, 7789, 8273, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=10240 expected_recv=16062 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8103, 8275, 7789, 8273, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8103, 8275, 7789, 8273, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=10240 expected_recv=16039 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8103, 8275, 7789, 8273, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=10240 expected_recv=16402 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=10240 expected_recv=16062 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=10240 expected_recv=16039 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8103, 8275, 7789, 8273, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7959, 8274, 8274, 7932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7959, 8274, 8274, 7932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=10240 expected_recv=16241 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7959, 8274, 8274, 7932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=10240 expected_recv=16234 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7959, 8274, 8274, 7932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7959, 8274, 8274, 7932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=10240 expected_recv=16241 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7959, 8274, 8274, 7932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=10240 expected_recv=16199 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7959, 8274, 8274, 7932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=10240 expected_recv=16234 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=10240 expected_recv=16207 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7959, 8274, 8274, 7932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=10240 expected_recv=16199 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=10240 expected_recv=16207 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8275, 8274, 8273, 7950, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8275, 8274, 8273, 7950, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8275, 8274, 8273, 7950, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8275, 8274, 8273, 7950, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=10240 expected_recv=15863 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8275, 8274, 8273, 7950, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8275, 8274, 8273, 7950, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=10240 expected_recv=15863 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8275, 8274, 8273, 7950, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=10240 expected_recv=15919 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8275, 8274, 8273, 7950, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=10240 expected_recv=15919 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=10240 expected_recv=16191 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8273, 8273, 8275, 7962, 8275, 7930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=10240 expected_recv=16191 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8273, 8273, 8275, 7962, 8275, 7930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8273, 8273, 8275, 7962, 8275, 7930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8273, 8273, 8275, 7962, 8275, 7930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8273, 8273, 8275, 7962, 8275, 7930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=10240 expected_recv=15936 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8273, 8273, 8275, 7962, 8275, 7930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8273, 8273, 8275, 7962, 8275, 7930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=10240 expected_recv=15936 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=10240 expected_recv=16204 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8273, 8273, 8275, 7962, 8275, 7930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=10240 expected_recv=16204 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7929, 7963, 8274, 8276, 8275, 8272, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7929, 7963, 8274, 8276, 8275, 8272, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=10240 expected_recv=15934 expected_overflow=0 exact_recv_per_rank=[7929, 7963, 8274, 8276, 8275, 8272, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7929, 7963, 8274, 8276, 8275, 8272, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7929, 7963, 8274, 8276, 8275, 8272, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=10240 expected_recv=15934 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7929, 7963, 8274, 8276, 8275, 8272, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[7929, 7963, 8274, 8276, 8275, 8272, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=10240 expected_recv=15851 expected_overflow=0 exact_recv_per_rank=[7929, 7963, 8274, 8276, 8275, 8272, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 8274, 8274, 7940, 8273, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=10240 expected_recv=15851 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 8274, 8274, 7940, 8273, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=10240 expected_recv=16188 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 8274, 8274, 7940, 8273, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 8274, 8274, 7940, 8273, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 8274, 8274, 7940, 8273, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=10240 expected_recv=16215 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 8274, 8274, 7940, 8273, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=10240 expected_recv=16188 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=10240 expected_recv=15929 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 8274, 8274, 7940, 8273, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=10240 expected_recv=16215 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=10240 expected_recv=14305 expected_overflow=0 exact_recv_per_rank=[7924, 8274, 8273, 8274, 8274, 7968, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=10240 expected_recv=15929 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 8274, 8274, 7940, 8273, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=10240 expected_recv=14305 actual_valid_recv=14592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=14592 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=10240 expected_recv=14247 expected_overflow=0 exact_recv_per_rank=[7924, 8274, 8273, 8274, 8274, 7968, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=10240 expected_recv=17204 expected_overflow=0 exact_recv_per_rank=[7924, 8274, 8273, 8274, 8274, 7968, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=10240 expected_recv=17204 expected_overflow=0 exact_recv_per_rank=[7924, 8274, 8273, 8274, 8274, 7968, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=10240 expected_recv=14247 actual_valid_recv=14592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=10240 expected_recv=17203 expected_overflow=0 exact_recv_per_rank=[7924, 8274, 8273, 8274, 8274, 7968, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=10240 expected_recv=16898 expected_overflow=0 exact_recv_per_rank=[7924, 8274, 8273, 8274, 8274, 7968, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=10240 expected_recv=17204 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=10240 expected_recv=17204 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=14592 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=10240 expected_recv=17203 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=10240 expected_recv=16898 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=10240 expected_recv=17205 expected_overflow=0 exact_recv_per_rank=[7924, 8274, 8273, 8274, 8274, 7968, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=17280 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=10240 expected_recv=17217 expected_overflow=0 exact_recv_per_rank=[8945, 8943, 8944, 8944, 8943, 5951, 5921, 8945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=17536 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=17280 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=17280 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=10240 expected_recv=17205 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=10240 expected_recv=17217 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=10240 expected_recv=16806 expected_overflow=0 exact_recv_per_rank=[7924, 8274, 8273, 8274, 8274, 7968, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=17536 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=17280 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=10240 expected_recv=16806 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=10240 expected_recv=16910 expected_overflow=0 exact_recv_per_rank=[8945, 8943, 8944, 8944, 8943, 5951, 5921, 8945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=10240 expected_recv=17217 expected_overflow=0 exact_recv_per_rank=[8945, 8943, 8944, 8944, 8943, 5951, 5921, 8945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=10240 expected_recv=16910 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=10240 expected_recv=17217 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=10240 expected_recv=14225 expected_overflow=0 exact_recv_per_rank=[8945, 8943, 8944, 8944, 8943, 5951, 5921, 8945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=10240 expected_recv=17218 expected_overflow=0 exact_recv_per_rank=[8945, 8943, 8944, 8944, 8943, 5951, 5921, 8945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=10240 expected_recv=14196 expected_overflow=0 exact_recv_per_rank=[8945, 8943, 8944, 8944, 8943, 5951, 5921, 8945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=17536 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7949, 8273, 7942, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=10240 expected_recv=14225 actual_valid_recv=14592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=10240 expected_recv=17218 expected_overflow=0 exact_recv_per_rank=[8945, 8943, 8944, 8944, 8943, 5951, 5921, 8945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=10240 expected_recv=17218 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=10240 expected_recv=14196 actual_valid_recv=14592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=10240 expected_recv=16871 expected_overflow=0 exact_recv_per_rank=[8945, 8943, 8944, 8944, 8943, 5951, 5921, 8945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=14592 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=17280 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=10240 expected_recv=17218 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=14592 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=10240 expected_recv=16871 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=17536 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=10240 expected_recv=16191 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7949, 8273, 7942, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=10240 expected_recv=16191 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7949, 8273, 7942, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=10240 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7949, 8273, 7942, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7949, 8273, 7942, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7949, 8273, 7942, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8275, 8274, 8274, 7932, 8273, 7961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=10240 expected_recv=16216 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7949, 8273, 7942, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=10240 expected_recv=15924 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7949, 8273, 7942, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=10240 expected_recv=15924 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8275, 8274, 8274, 7932, 8273, 7961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8275, 8274, 8274, 7932, 8273, 7961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8275, 8274, 8274, 7932, 8273, 7961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=10240 expected_recv=16206 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8275, 8274, 8274, 7932, 8273, 7961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=10240 expected_recv=15919 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8275, 8274, 8274, 7932, 8273, 7961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=10240 expected_recv=16187 expected_overflow=0 exact_recv_per_rank=[7927, 8275, 8274, 8274, 8275, 8273, 7965, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=10240 expected_recv=16208 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8275, 8274, 8274, 7932, 8273, 7961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=10240 expected_recv=16206 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8275, 8274, 8274, 7932, 8273, 7961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=10240 expected_recv=15919 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=10240 expected_recv=16187 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=10240 expected_recv=16208 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7927, 8275, 8274, 8274, 8275, 8273, 7965, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7927, 8275, 8274, 8274, 8275, 8273, 7965, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=10240 expected_recv=15944 expected_overflow=0 exact_recv_per_rank=[7927, 8275, 8274, 8274, 8275, 8273, 7965, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7927, 8275, 8274, 8274, 8275, 8273, 7965, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7927, 8275, 8274, 8274, 8275, 8273, 7965, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=10240 expected_recv=15944 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7927, 8275, 8274, 8274, 8275, 8273, 7965, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=10240 expected_recv=16202 expected_overflow=0 exact_recv_per_rank=[7927, 8275, 8274, 8274, 8275, 8273, 7965, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[304, 309, 2921, 5781, 10327, 6539, 8784, 30571] avg=8192.0000 max=30571 max_rank=7 min=304 min_rank=0 imbalance_ratio=3.731812 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=10240 expected_recv=16202 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[304, 309, 2921, 5781, 10327, 6539, 8784, 30571] avg=8192.0000 max=30571 max_rank=7 min=304 min_rank=0 imbalance_ratio=3.731812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[8580, 8580, 8365, 7528, 8580, 6743, 8580, 8580] avg=8192.0000 max=8580 max_rank=0 min=6743 min_rank=5 imbalance_ratio=1.047363 max_load_reduction=21991 max_load_reduction_pct=0.719342 moved_to_replicas=48542 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[304, 309, 2921, 5781, 10327, 6539, 8784, 30571] avg=8192.0000 max=30571 max_rank=7 min=304 min_rank=0 imbalance_ratio=3.731812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[304, 309, 2921, 5781, 10327, 6539, 8784, 30571] avg=8192.0000 max=30571 max_rank=7 min=304 min_rank=0 imbalance_ratio=3.731812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[304, 309, 2921, 5781, 10327, 6539, 8784, 30571] avg=8192.0000 max=30571 max_rank=7 min=304 min_rank=0 imbalance_ratio=3.731812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[304, 309, 2921, 5781, 10327, 6539, 8784, 30571] avg=8192.0000 max=30571 max_rank=7 min=304 min_rank=0 imbalance_ratio=3.731812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[8580, 8580, 8365, 7528, 8580, 6743, 8580, 8580] avg=8192.0000 max=8580 max_rank=0 min=6743 min_rank=5 imbalance_ratio=1.047363 max_load_reduction=21991 max_load_reduction_pct=0.719342 moved_to_replicas=48542 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[304, 309, 2921, 5781, 10327, 6539, 8784, 30571] avg=8192.0000 max=30571 max_rank=7 min=304 min_rank=0 imbalance_ratio=3.731812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[304, 309, 2921, 5781, 10327, 6539, 8784, 30571] avg=8192.0000 max=30571 max_rank=7 min=304 min_rank=0 imbalance_ratio=3.731812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[8580, 8580, 8365, 7528, 8580, 6743, 8580, 8580] avg=8192.0000 max=8580 max_rank=0 min=6743 min_rank=5 imbalance_ratio=1.047363 max_load_reduction=21991 max_load_reduction_pct=0.719342 moved_to_replicas=48542 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[8580, 8580, 8365, 7528, 8580, 6743, 8580, 8580] avg=8192.0000 max=8580 max_rank=0 min=6743 min_rank=5 imbalance_ratio=1.047363 max_load_reduction=21991 max_load_reduction_pct=0.719342 moved_to_replicas=48542 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[25178, 2, 12, 1, 7983, 151, 50, 32159] avg=8192.0000 max=32159 max_rank=7 min=1 min_rank=3 imbalance_ratio=3.925659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[8580, 8580, 8365, 7528, 8580, 6743, 8580, 8580] avg=8192.0000 max=8580 max_rank=0 min=6743 min_rank=5 imbalance_ratio=1.047363 max_load_reduction=21991 max_load_reduction_pct=0.719342 moved_to_replicas=48542 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[8580, 8580, 8365, 7528, 8580, 6743, 8580, 8580] avg=8192.0000 max=8580 max_rank=0 min=6743 min_rank=5 imbalance_ratio=1.047363 max_load_reduction=21991 max_load_reduction_pct=0.719342 moved_to_replicas=48542 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[8580, 8580, 8365, 7528, 8580, 6743, 8580, 8580] avg=8192.0000 max=8580 max_rank=0 min=6743 min_rank=5 imbalance_ratio=1.047363 max_load_reduction=21991 max_load_reduction_pct=0.719342 moved_to_replicas=48542 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[8580, 8580, 8365, 7528, 8580, 6743, 8580, 8580] avg=8192.0000 max=8580 max_rank=0 min=6743 min_rank=5 imbalance_ratio=1.047363 max_load_reduction=21991 max_load_reduction_pct=0.719342 moved_to_replicas=48542 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[25178, 2, 12, 1, 7983, 151, 50, 32159] avg=8192.0000 max=32159 max_rank=7 min=1 min_rank=3 imbalance_ratio=3.925659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[8596, 8595, 6386, 8596, 7983, 8187, 8596, 8597] avg=8192.0000 max=8597 max_rank=7 min=6386 min_rank=2 imbalance_ratio=1.049438 max_load_reduction=23562 max_load_reduction_pct=0.732672 moved_to_replicas=80050 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[25178, 2, 12, 1, 7983, 151, 50, 32159] avg=8192.0000 max=32159 max_rank=7 min=1 min_rank=3 imbalance_ratio=3.925659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[25178, 2, 12, 1, 7983, 151, 50, 32159] avg=8192.0000 max=32159 max_rank=7 min=1 min_rank=3 imbalance_ratio=3.925659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[25178, 2, 12, 1, 7983, 151, 50, 32159] avg=8192.0000 max=32159 max_rank=7 min=1 min_rank=3 imbalance_ratio=3.925659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[8596, 8595, 6386, 8596, 7983, 8187, 8596, 8597] avg=8192.0000 max=8597 max_rank=7 min=6386 min_rank=2 imbalance_ratio=1.049438 max_load_reduction=23562 max_load_reduction_pct=0.732672 moved_to_replicas=80050 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[25178, 2, 12, 1, 7983, 151, 50, 32159] avg=8192.0000 max=32159 max_rank=7 min=1 min_rank=3 imbalance_ratio=3.925659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[25178, 2, 12, 1, 7983, 151, 50, 32159] avg=8192.0000 max=32159 max_rank=7 min=1 min_rank=3 imbalance_ratio=3.925659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[25178, 2, 12, 1, 7983, 151, 50, 32159] avg=8192.0000 max=32159 max_rank=7 min=1 min_rank=3 imbalance_ratio=3.925659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[8596, 8595, 6386, 8596, 7983, 8187, 8596, 8597] avg=8192.0000 max=8597 max_rank=7 min=6386 min_rank=2 imbalance_ratio=1.049438 max_load_reduction=23562 max_load_reduction_pct=0.732672 moved_to_replicas=80050 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[32631, 1839, 69, 3754, 21, 2996, 24222, 4] avg=8192.0000 max=32631 max_rank=0 min=4 min_rank=7 imbalance_ratio=3.983276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[8596, 8595, 6386, 8596, 7983, 8187, 8596, 8597] avg=8192.0000 max=8597 max_rank=7 min=6386 min_rank=2 imbalance_ratio=1.049438 max_load_reduction=23562 max_load_reduction_pct=0.732672 moved_to_replicas=80050 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[8596, 8595, 6386, 8596, 7983, 8187, 8596, 8597] avg=8192.0000 max=8597 max_rank=7 min=6386 min_rank=2 imbalance_ratio=1.049438 max_load_reduction=23562 max_load_reduction_pct=0.732672 moved_to_replicas=80050 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[8596, 8595, 6386, 8596, 7983, 8187, 8596, 8597] avg=8192.0000 max=8597 max_rank=7 min=6386 min_rank=2 imbalance_ratio=1.049438 max_load_reduction=23562 max_load_reduction_pct=0.732672 moved_to_replicas=80050 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[8596, 8595, 6386, 8596, 7983, 8187, 8596, 8597] avg=8192.0000 max=8597 max_rank=7 min=6386 min_rank=2 imbalance_ratio=1.049438 max_load_reduction=23562 max_load_reduction_pct=0.732672 moved_to_replicas=80050 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[8596, 8595, 6386, 8596, 7983, 8187, 8596, 8597] avg=8192.0000 max=8597 max_rank=7 min=6386 min_rank=2 imbalance_ratio=1.049438 max_load_reduction=23562 max_load_reduction_pct=0.732672 moved_to_replicas=80050 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[32631, 1839, 69, 3754, 21, 2996, 24222, 4] avg=8192.0000 max=32631 max_rank=0 min=4 min_rank=7 imbalance_ratio=3.983276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[8274, 8275, 7903, 7988, 8274, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=1 min=7903 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24356 max_load_reduction_pct=0.746407 moved_to_replicas=78849 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[32631, 1839, 69, 3754, 21, 2996, 24222, 4] avg=8192.0000 max=32631 max_rank=0 min=4 min_rank=7 imbalance_ratio=3.983276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[32631, 1839, 69, 3754, 21, 2996, 24222, 4] avg=8192.0000 max=32631 max_rank=0 min=4 min_rank=7 imbalance_ratio=3.983276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[32631, 1839, 69, 3754, 21, 2996, 24222, 4] avg=8192.0000 max=32631 max_rank=0 min=4 min_rank=7 imbalance_ratio=3.983276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[8274, 8275, 7903, 7988, 8274, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=1 min=7903 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24356 max_load_reduction_pct=0.746407 moved_to_replicas=78849 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[32631, 1839, 69, 3754, 21, 2996, 24222, 4] avg=8192.0000 max=32631 max_rank=0 min=4 min_rank=7 imbalance_ratio=3.983276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[32631, 1839, 69, 3754, 21, 2996, 24222, 4] avg=8192.0000 max=32631 max_rank=0 min=4 min_rank=7 imbalance_ratio=3.983276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[32631, 1839, 69, 3754, 21, 2996, 24222, 4] avg=8192.0000 max=32631 max_rank=0 min=4 min_rank=7 imbalance_ratio=3.983276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[4884, 44, 24075, 21, 5, 4, 4908, 31595] avg=8192.0000 max=31595 max_rank=7 min=4 min_rank=5 imbalance_ratio=3.856812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[8274, 8275, 7903, 7988, 8274, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=1 min=7903 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24356 max_load_reduction_pct=0.746407 moved_to_replicas=78849 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[8274, 8275, 7903, 7988, 8274, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=1 min=7903 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24356 max_load_reduction_pct=0.746407 moved_to_replicas=78849 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[8274, 8275, 7903, 7988, 8274, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=1 min=7903 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24356 max_load_reduction_pct=0.746407 moved_to_replicas=78849 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[8274, 8275, 7903, 7988, 8274, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=1 min=7903 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24356 max_load_reduction_pct=0.746407 moved_to_replicas=78849 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[8274, 8275, 7903, 7988, 8274, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=1 min=7903 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24356 max_load_reduction_pct=0.746407 moved_to_replicas=78849 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[8590, 8590, 8589, 5855, 8590, 8590, 8142, 8590] avg=8192.0000 max=8590 max_rank=0 min=5855 min_rank=3 imbalance_ratio=1.048584 max_load_reduction=23005 max_load_reduction_pct=0.728122 moved_to_replicas=76656 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[8274, 8275, 7903, 7988, 8274, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=1 min=7903 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24356 max_load_reduction_pct=0.746407 moved_to_replicas=78849 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[4884, 44, 24075, 21, 5, 4, 4908, 31595] avg=8192.0000 max=31595 max_rank=7 min=4 min_rank=5 imbalance_ratio=3.856812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[4884, 44, 24075, 21, 5, 4, 4908, 31595] avg=8192.0000 max=31595 max_rank=7 min=4 min_rank=5 imbalance_ratio=3.856812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[4884, 44, 24075, 21, 5, 4, 4908, 31595] avg=8192.0000 max=31595 max_rank=7 min=4 min_rank=5 imbalance_ratio=3.856812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[4884, 44, 24075, 21, 5, 4, 4908, 31595] avg=8192.0000 max=31595 max_rank=7 min=4 min_rank=5 imbalance_ratio=3.856812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[4884, 44, 24075, 21, 5, 4, 4908, 31595] avg=8192.0000 max=31595 max_rank=7 min=4 min_rank=5 imbalance_ratio=3.856812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[79, 8634, 27, 8, 7, 24105, 4, 32672] avg=8192.0000 max=32672 max_rank=7 min=4 min_rank=6 imbalance_ratio=3.988281 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[8590, 8590, 8589, 5855, 8590, 8590, 8142, 8590] avg=8192.0000 max=8590 max_rank=0 min=5855 min_rank=3 imbalance_ratio=1.048584 max_load_reduction=23005 max_load_reduction_pct=0.728122 moved_to_replicas=76656 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[8590, 8590, 8589, 5855, 8590, 8590, 8142, 8590] avg=8192.0000 max=8590 max_rank=0 min=5855 min_rank=3 imbalance_ratio=1.048584 max_load_reduction=23005 max_load_reduction_pct=0.728122 moved_to_replicas=76656 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[4884, 44, 24075, 21, 5, 4, 4908, 31595] avg=8192.0000 max=31595 max_rank=7 min=4 min_rank=5 imbalance_ratio=3.856812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[8590, 8590, 8589, 5855, 8590, 8590, 8142, 8590] avg=8192.0000 max=8590 max_rank=0 min=5855 min_rank=3 imbalance_ratio=1.048584 max_load_reduction=23005 max_load_reduction_pct=0.728122 moved_to_replicas=76656 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[4884, 44, 24075, 21, 5, 4, 4908, 31595] avg=8192.0000 max=31595 max_rank=7 min=4 min_rank=5 imbalance_ratio=3.856812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[8590, 8590, 8589, 5855, 8590, 8590, 8142, 8590] avg=8192.0000 max=8590 max_rank=0 min=5855 min_rank=3 imbalance_ratio=1.048584 max_load_reduction=23005 max_load_reduction_pct=0.728122 moved_to_replicas=76656 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[8590, 8590, 8589, 5855, 8590, 8590, 8142, 8590] avg=8192.0000 max=8590 max_rank=0 min=5855 min_rank=3 imbalance_ratio=1.048584 max_load_reduction=23005 max_load_reduction_pct=0.728122 moved_to_replicas=76656 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[6355, 8634, 8927, 5908, 8927, 8929, 8928, 8928] avg=8192.0000 max=8929 max_rank=5 min=5908 min_rank=3 imbalance_ratio=1.089966 max_load_reduction=23743 max_load_reduction_pct=0.726708 moved_to_replicas=71324 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[8590, 8590, 8589, 5855, 8590, 8590, 8142, 8590] avg=8192.0000 max=8590 max_rank=0 min=5855 min_rank=3 imbalance_ratio=1.048584 max_load_reduction=23005 max_load_reduction_pct=0.728122 moved_to_replicas=76656 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[8590, 8590, 8589, 5855, 8590, 8590, 8142, 8590] avg=8192.0000 max=8590 max_rank=0 min=5855 min_rank=3 imbalance_ratio=1.048584 max_load_reduction=23005 max_load_reduction_pct=0.728122 moved_to_replicas=76656 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[79, 8634, 27, 8, 7, 24105, 4, 32672] avg=8192.0000 max=32672 max_rank=7 min=4 min_rank=6 imbalance_ratio=3.988281 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[79, 8634, 27, 8, 7, 24105, 4, 32672] avg=8192.0000 max=32672 max_rank=7 min=4 min_rank=6 imbalance_ratio=3.988281 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[79, 8634, 27, 8, 7, 24105, 4, 32672] avg=8192.0000 max=32672 max_rank=7 min=4 min_rank=6 imbalance_ratio=3.988281 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[79, 8634, 27, 8, 7, 24105, 4, 32672] avg=8192.0000 max=32672 max_rank=7 min=4 min_rank=6 imbalance_ratio=3.988281 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[79, 8634, 27, 8, 7, 24105, 4, 32672] avg=8192.0000 max=32672 max_rank=7 min=4 min_rank=6 imbalance_ratio=3.988281 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[29718, 13, 1714, 32606, 4, 2, 1479, 0] avg=8192.0000 max=32606 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.980225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[6355, 8634, 8927, 5908, 8927, 8929, 8928, 8928] avg=8192.0000 max=8929 max_rank=5 min=5908 min_rank=3 imbalance_ratio=1.089966 max_load_reduction=23743 max_load_reduction_pct=0.726708 moved_to_replicas=71324 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[6355, 8634, 8927, 5908, 8927, 8929, 8928, 8928] avg=8192.0000 max=8929 max_rank=5 min=5908 min_rank=3 imbalance_ratio=1.089966 max_load_reduction=23743 max_load_reduction_pct=0.726708 moved_to_replicas=71324 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[79, 8634, 27, 8, 7, 24105, 4, 32672] avg=8192.0000 max=32672 max_rank=7 min=4 min_rank=6 imbalance_ratio=3.988281 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[6355, 8634, 8927, 5908, 8927, 8929, 8928, 8928] avg=8192.0000 max=8929 max_rank=5 min=5908 min_rank=3 imbalance_ratio=1.089966 max_load_reduction=23743 max_load_reduction_pct=0.726708 moved_to_replicas=71324 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[6355, 8634, 8927, 5908, 8927, 8929, 8928, 8928] avg=8192.0000 max=8929 max_rank=5 min=5908 min_rank=3 imbalance_ratio=1.089966 max_load_reduction=23743 max_load_reduction_pct=0.726708 moved_to_replicas=71324 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[79, 8634, 27, 8, 7, 24105, 4, 32672] avg=8192.0000 max=32672 max_rank=7 min=4 min_rank=6 imbalance_ratio=3.988281 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[6355, 8634, 8927, 5908, 8927, 8929, 8928, 8928] avg=8192.0000 max=8929 max_rank=5 min=5908 min_rank=3 imbalance_ratio=1.089966 max_load_reduction=23743 max_load_reduction_pct=0.726708 moved_to_replicas=71324 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[6355, 8634, 8927, 5908, 8927, 8929, 8928, 8928] avg=8192.0000 max=8929 max_rank=5 min=5908 min_rank=3 imbalance_ratio=1.089966 max_load_reduction=23743 max_load_reduction_pct=0.726708 moved_to_replicas=71324 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[8274, 8273, 8103, 8275, 7789, 8273, 8274, 8275] avg=8192.0000 max=8275 max_rank=3 min=7789 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24331 max_load_reduction_pct=0.746212 moved_to_replicas=92024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[6355, 8634, 8927, 5908, 8927, 8929, 8928, 8928] avg=8192.0000 max=8929 max_rank=5 min=5908 min_rank=3 imbalance_ratio=1.089966 max_load_reduction=23743 max_load_reduction_pct=0.726708 moved_to_replicas=71324 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[29718, 13, 1714, 32606, 4, 2, 1479, 0] avg=8192.0000 max=32606 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.980225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[29718, 13, 1714, 32606, 4, 2, 1479, 0] avg=8192.0000 max=32606 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.980225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[29718, 13, 1714, 32606, 4, 2, 1479, 0] avg=8192.0000 max=32606 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.980225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[29718, 13, 1714, 32606, 4, 2, 1479, 0] avg=8192.0000 max=32606 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.980225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[29718, 13, 1714, 32606, 4, 2, 1479, 0] avg=8192.0000 max=32606 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.980225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[8274, 8273, 8103, 8275, 7789, 8273, 8274, 8275] avg=8192.0000 max=8275 max_rank=3 min=7789 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24331 max_load_reduction_pct=0.746212 moved_to_replicas=92024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[8274, 8273, 8103, 8275, 7789, 8273, 8274, 8275] avg=8192.0000 max=8275 max_rank=3 min=7789 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24331 max_load_reduction_pct=0.746212 moved_to_replicas=92024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[29718, 13, 1714, 32606, 4, 2, 1479, 0] avg=8192.0000 max=32606 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.980225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[8274, 8273, 8103, 8275, 7789, 8273, 8274, 8275] avg=8192.0000 max=8275 max_rank=3 min=7789 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24331 max_load_reduction_pct=0.746212 moved_to_replicas=92024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[8274, 8273, 8103, 8275, 7789, 8273, 8274, 8275] avg=8192.0000 max=8275 max_rank=3 min=7789 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24331 max_load_reduction_pct=0.746212 moved_to_replicas=92024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[16, 6, 2, 32729, 31, 4, 32743, 5] avg=8192.0000 max=32743 max_rank=6 min=2 min_rank=2 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[8274, 8273, 8103, 8275, 7789, 8273, 8274, 8275] avg=8192.0000 max=8275 max_rank=3 min=7789 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24331 max_load_reduction_pct=0.746212 moved_to_replicas=92024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[29718, 13, 1714, 32606, 4, 2, 1479, 0] avg=8192.0000 max=32606 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.980225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[8274, 8273, 8103, 8275, 7789, 8273, 8274, 8275] avg=8192.0000 max=8275 max_rank=3 min=7789 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24331 max_load_reduction_pct=0.746212 moved_to_replicas=92024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[16, 6, 2, 32729, 31, 4, 32743, 5] avg=8192.0000 max=32743 max_rank=6 min=2 min_rank=2 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7959, 8274, 8274, 7932] avg=8192.0000 max=8275 max_rank=1 min=7932 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24468 max_load_reduction_pct=0.747274 moved_to_replicas=97786 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[8274, 8273, 8103, 8275, 7789, 8273, 8274, 8275] avg=8192.0000 max=8275 max_rank=3 min=7789 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24331 max_load_reduction_pct=0.746212 moved_to_replicas=92024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[16, 6, 2, 32729, 31, 4, 32743, 5] avg=8192.0000 max=32743 max_rank=6 min=2 min_rank=2 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[16, 6, 2, 32729, 31, 4, 32743, 5] avg=8192.0000 max=32743 max_rank=6 min=2 min_rank=2 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[16, 6, 2, 32729, 31, 4, 32743, 5] avg=8192.0000 max=32743 max_rank=6 min=2 min_rank=2 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[16, 6, 2, 32729, 31, 4, 32743, 5] avg=8192.0000 max=32743 max_rank=6 min=2 min_rank=2 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7959, 8274, 8274, 7932] avg=8192.0000 max=8275 max_rank=1 min=7932 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24468 max_load_reduction_pct=0.747274 moved_to_replicas=97786 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7959, 8274, 8274, 7932] avg=8192.0000 max=8275 max_rank=1 min=7932 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24468 max_load_reduction_pct=0.747274 moved_to_replicas=97786 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[16, 6, 2, 32729, 31, 4, 32743, 5] avg=8192.0000 max=32743 max_rank=6 min=2 min_rank=2 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7959, 8274, 8274, 7932] avg=8192.0000 max=8275 max_rank=1 min=7932 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24468 max_load_reduction_pct=0.747274 moved_to_replicas=97786 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[4, 5605, 5, 32753, 17617, 1621, 11, 7920] avg=8192.0000 max=32753 max_rank=3 min=4 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7959, 8274, 8274, 7932] avg=8192.0000 max=8275 max_rank=1 min=7932 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24468 max_load_reduction_pct=0.747274 moved_to_replicas=97786 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7959, 8274, 8274, 7932] avg=8192.0000 max=8275 max_rank=1 min=7932 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24468 max_load_reduction_pct=0.747274 moved_to_replicas=97786 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[16, 6, 2, 32729, 31, 4, 32743, 5] avg=8192.0000 max=32743 max_rank=6 min=2 min_rank=2 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7959, 8274, 8274, 7932] avg=8192.0000 max=8275 max_rank=1 min=7932 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24468 max_load_reduction_pct=0.747274 moved_to_replicas=97786 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[4, 5605, 5, 32753, 17617, 1621, 11, 7920] avg=8192.0000 max=32753 max_rank=3 min=4 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[8274, 8275, 8274, 8275, 8274, 8273, 7950, 7941] avg=8192.0000 max=8275 max_rank=1 min=7941 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=69764 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7959, 8274, 8274, 7932] avg=8192.0000 max=8275 max_rank=1 min=7932 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24468 max_load_reduction_pct=0.747274 moved_to_replicas=97786 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[4, 5605, 5, 32753, 17617, 1621, 11, 7920] avg=8192.0000 max=32753 max_rank=3 min=4 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[4, 5605, 5, 32753, 17617, 1621, 11, 7920] avg=8192.0000 max=32753 max_rank=3 min=4 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[4, 5605, 5, 32753, 17617, 1621, 11, 7920] avg=8192.0000 max=32753 max_rank=3 min=4 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[4, 5605, 5, 32753, 17617, 1621, 11, 7920] avg=8192.0000 max=32753 max_rank=3 min=4 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[8274, 8275, 8274, 8275, 8274, 8273, 7950, 7941] avg=8192.0000 max=8275 max_rank=1 min=7941 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=69764 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[8274, 8275, 8274, 8275, 8274, 8273, 7950, 7941] avg=8192.0000 max=8275 max_rank=1 min=7941 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=69764 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[4, 5605, 5, 32753, 17617, 1621, 11, 7920] avg=8192.0000 max=32753 max_rank=3 min=4 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[8274, 8275, 8274, 8275, 8274, 8273, 7950, 7941] avg=8192.0000 max=8275 max_rank=1 min=7941 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=69764 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[8274, 8275, 8274, 8275, 8274, 8273, 7950, 7941] avg=8192.0000 max=8275 max_rank=1 min=7941 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=69764 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[8274, 8275, 8274, 8275, 8274, 8273, 7950, 7941] avg=8192.0000 max=8275 max_rank=1 min=7941 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=69764 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[32715, 5, 5, 9, 17, 43, 32734, 8] avg=8192.0000 max=32734 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[4, 5605, 5, 32753, 17617, 1621, 11, 7920] avg=8192.0000 max=32753 max_rank=3 min=4 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[8274, 8275, 8274, 8275, 8274, 8273, 7950, 7941] avg=8192.0000 max=8275 max_rank=1 min=7941 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=69764 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[32715, 5, 5, 9, 17, 43, 32734, 8] avg=8192.0000 max=32734 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[8274, 8274, 8273, 8273, 8275, 7962, 8275, 7930] avg=8192.0000 max=8275 max_rank=4 min=7930 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24459 max_load_reduction_pct=0.747205 moved_to_replicas=97702 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[32715, 5, 5, 9, 17, 43, 32734, 8] avg=8192.0000 max=32734 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[8274, 8275, 8274, 8275, 8274, 8273, 7950, 7941] avg=8192.0000 max=8275 max_rank=1 min=7941 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=69764 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[32715, 5, 5, 9, 17, 43, 32734, 8] avg=8192.0000 max=32734 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[32715, 5, 5, 9, 17, 43, 32734, 8] avg=8192.0000 max=32734 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[32715, 5, 5, 9, 17, 43, 32734, 8] avg=8192.0000 max=32734 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[8274, 8274, 8273, 8273, 8275, 7962, 8275, 7930] avg=8192.0000 max=8275 max_rank=4 min=7930 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24459 max_load_reduction_pct=0.747205 moved_to_replicas=97702 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[8274, 8274, 8273, 8273, 8275, 7962, 8275, 7930] avg=8192.0000 max=8275 max_rank=4 min=7930 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24459 max_load_reduction_pct=0.747205 moved_to_replicas=97702 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[8274, 8274, 8273, 8273, 8275, 7962, 8275, 7930] avg=8192.0000 max=8275 max_rank=4 min=7930 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24459 max_load_reduction_pct=0.747205 moved_to_replicas=97702 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[32715, 5, 5, 9, 17, 43, 32734, 8] avg=8192.0000 max=32734 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[8274, 8274, 8273, 8273, 8275, 7962, 8275, 7930] avg=8192.0000 max=8275 max_rank=4 min=7930 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24459 max_load_reduction_pct=0.747205 moved_to_replicas=97702 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[8274, 8274, 8273, 8273, 8275, 7962, 8275, 7930] avg=8192.0000 max=8275 max_rank=4 min=7930 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24459 max_load_reduction_pct=0.747205 moved_to_replicas=97702 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[6, 5036, 32741, 2, 7, 2, 27725, 17] avg=8192.0000 max=32741 max_rank=2 min=2 min_rank=3 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[32715, 5, 5, 9, 17, 43, 32734, 8] avg=8192.0000 max=32734 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[6, 5036, 32741, 2, 7, 2, 27725, 17] avg=8192.0000 max=32741 max_rank=2 min=2 min_rank=3 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[8274, 8274, 8273, 8273, 8275, 7962, 8275, 7930] avg=8192.0000 max=8275 max_rank=4 min=7930 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24459 max_load_reduction_pct=0.747205 moved_to_replicas=97702 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[7929, 7963, 8274, 8276, 8275, 8272, 8274, 8273] avg=8192.0000 max=8276 max_rank=3 min=7929 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24465 max_load_reduction_pct=0.747228 moved_to_replicas=88637 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[6, 5036, 32741, 2, 7, 2, 27725, 17] avg=8192.0000 max=32741 max_rank=2 min=2 min_rank=3 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[8274, 8274, 8273, 8273, 8275, 7962, 8275, 7930] avg=8192.0000 max=8275 max_rank=4 min=7930 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24459 max_load_reduction_pct=0.747205 moved_to_replicas=97702 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[6, 5036, 32741, 2, 7, 2, 27725, 17] avg=8192.0000 max=32741 max_rank=2 min=2 min_rank=3 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[6, 5036, 32741, 2, 7, 2, 27725, 17] avg=8192.0000 max=32741 max_rank=2 min=2 min_rank=3 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[6, 5036, 32741, 2, 7, 2, 27725, 17] avg=8192.0000 max=32741 max_rank=2 min=2 min_rank=3 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[7929, 7963, 8274, 8276, 8275, 8272, 8274, 8273] avg=8192.0000 max=8276 max_rank=3 min=7929 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24465 max_load_reduction_pct=0.747228 moved_to_replicas=88637 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[7929, 7963, 8274, 8276, 8275, 8272, 8274, 8273] avg=8192.0000 max=8276 max_rank=3 min=7929 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24465 max_load_reduction_pct=0.747228 moved_to_replicas=88637 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[7929, 7963, 8274, 8276, 8275, 8272, 8274, 8273] avg=8192.0000 max=8276 max_rank=3 min=7929 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24465 max_load_reduction_pct=0.747228 moved_to_replicas=88637 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[6, 5036, 32741, 2, 7, 2, 27725, 17] avg=8192.0000 max=32741 max_rank=2 min=2 min_rank=3 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[7929, 7963, 8274, 8276, 8275, 8272, 8274, 8273] avg=8192.0000 max=8276 max_rank=3 min=7929 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24465 max_load_reduction_pct=0.747228 moved_to_replicas=88637 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[18, 9, 32629, 32741, 2, 10, 15, 112] avg=8192.0000 max=32741 max_rank=3 min=2 min_rank=4 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[7929, 7963, 8274, 8276, 8275, 8272, 8274, 8273] avg=8192.0000 max=8276 max_rank=3 min=7929 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24465 max_load_reduction_pct=0.747228 moved_to_replicas=88637 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[6, 5036, 32741, 2, 7, 2, 27725, 17] avg=8192.0000 max=32741 max_rank=2 min=2 min_rank=3 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[18, 9, 32629, 32741, 2, 10, 15, 112] avg=8192.0000 max=32741 max_rank=3 min=2 min_rank=4 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[7929, 7963, 8274, 8276, 8275, 8272, 8274, 8273] avg=8192.0000 max=8276 max_rank=3 min=7929 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24465 max_load_reduction_pct=0.747228 moved_to_replicas=88637 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[8274, 8274, 8275, 8274, 8274, 7940, 8273, 7952] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24466 max_load_reduction_pct=0.747259 moved_to_replicas=97597 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[18, 9, 32629, 32741, 2, 10, 15, 112] avg=8192.0000 max=32741 max_rank=3 min=2 min_rank=4 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[7929, 7963, 8274, 8276, 8275, 8272, 8274, 8273] avg=8192.0000 max=8276 max_rank=3 min=7929 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24465 max_load_reduction_pct=0.747228 moved_to_replicas=88637 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[18, 9, 32629, 32741, 2, 10, 15, 112] avg=8192.0000 max=32741 max_rank=3 min=2 min_rank=4 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[18, 9, 32629, 32741, 2, 10, 15, 112] avg=8192.0000 max=32741 max_rank=3 min=2 min_rank=4 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[18, 9, 32629, 32741, 2, 10, 15, 112] avg=8192.0000 max=32741 max_rank=3 min=2 min_rank=4 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[8274, 8274, 8275, 8274, 8274, 7940, 8273, 7952] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24466 max_load_reduction_pct=0.747259 moved_to_replicas=97597 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[8274, 8274, 8275, 8274, 8274, 7940, 8273, 7952] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24466 max_load_reduction_pct=0.747259 moved_to_replicas=97597 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[8274, 8274, 8275, 8274, 8274, 7940, 8273, 7952] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24466 max_load_reduction_pct=0.747259 moved_to_replicas=97597 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[7095, 25587, 17, 41, 32733, 23, 23, 17] avg=8192.0000 max=32733 max_rank=4 min=17 min_rank=2 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[18, 9, 32629, 32741, 2, 10, 15, 112] avg=8192.0000 max=32741 max_rank=3 min=2 min_rank=4 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[8274, 8274, 8275, 8274, 8274, 7940, 8273, 7952] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24466 max_load_reduction_pct=0.747259 moved_to_replicas=97597 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[8274, 8274, 8275, 8274, 8274, 7940, 8273, 7952] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24466 max_load_reduction_pct=0.747259 moved_to_replicas=97597 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[18, 9, 32629, 32741, 2, 10, 15, 112] avg=8192.0000 max=32741 max_rank=3 min=2 min_rank=4 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[7095, 25587, 17, 41, 32733, 23, 23, 17] avg=8192.0000 max=32733 max_rank=4 min=17 min_rank=2 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[8274, 8274, 8275, 8274, 8274, 7940, 8273, 7952] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24466 max_load_reduction_pct=0.747259 moved_to_replicas=97597 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[7924, 8274, 8273, 8274, 8274, 7968, 8274, 8275] avg=8192.0000 max=8275 max_rank=7 min=7924 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24458 max_load_reduction_pct=0.747197 moved_to_replicas=80433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[7095, 25587, 17, 41, 32733, 23, 23, 17] avg=8192.0000 max=32733 max_rank=4 min=17 min_rank=2 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[7095, 25587, 17, 41, 32733, 23, 23, 17] avg=8192.0000 max=32733 max_rank=4 min=17 min_rank=2 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[8274, 8274, 8275, 8274, 8274, 7940, 8273, 7952] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24466 max_load_reduction_pct=0.747259 moved_to_replicas=97597 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[7924, 8274, 8273, 8274, 8274, 7968, 8274, 8275] avg=8192.0000 max=8275 max_rank=7 min=7924 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24458 max_load_reduction_pct=0.747197 moved_to_replicas=80433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[7095, 25587, 17, 41, 32733, 23, 23, 17] avg=8192.0000 max=32733 max_rank=4 min=17 min_rank=2 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[7095, 25587, 17, 41, 32733, 23, 23, 17] avg=8192.0000 max=32733 max_rank=4 min=17 min_rank=2 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[7924, 8274, 8273, 8274, 8274, 7968, 8274, 8275] avg=8192.0000 max=8275 max_rank=7 min=7924 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24458 max_load_reduction_pct=0.747197 moved_to_replicas=80433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[7924, 8274, 8273, 8274, 8274, 7968, 8274, 8275] avg=8192.0000 max=8275 max_rank=7 min=7924 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24458 max_load_reduction_pct=0.747197 moved_to_replicas=80433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[0, 19, 8944, 23797, 7, 22, 9, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[7095, 25587, 17, 41, 32733, 23, 23, 17] avg=8192.0000 max=32733 max_rank=4 min=17 min_rank=2 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[7924, 8274, 8273, 8274, 8274, 7968, 8274, 8275] avg=8192.0000 max=8275 max_rank=7 min=7924 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24458 max_load_reduction_pct=0.747197 moved_to_replicas=80433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[7924, 8274, 8273, 8274, 8274, 7968, 8274, 8275] avg=8192.0000 max=8275 max_rank=7 min=7924 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24458 max_load_reduction_pct=0.747197 moved_to_replicas=80433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[7095, 25587, 17, 41, 32733, 23, 23, 17] avg=8192.0000 max=32733 max_rank=4 min=17 min_rank=2 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[0, 19, 8944, 23797, 7, 22, 9, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[8945, 8943, 8944, 8944, 8943, 5951, 5921, 8945] avg=8192.0000 max=8945 max_rank=0 min=5921 min_rank=6 imbalance_ratio=1.091919 max_load_reduction=23793 max_load_reduction_pct=0.726770 moved_to_replicas=80049 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[7924, 8274, 8273, 8274, 8274, 7968, 8274, 8275] avg=8192.0000 max=8275 max_rank=7 min=7924 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24458 max_load_reduction_pct=0.747197 moved_to_replicas=80433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[0, 19, 8944, 23797, 7, 22, 9, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[0, 19, 8944, 23797, 7, 22, 9, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[7924, 8274, 8273, 8274, 8274, 7968, 8274, 8275] avg=8192.0000 max=8275 max_rank=7 min=7924 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24458 max_load_reduction_pct=0.747197 moved_to_replicas=80433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[0, 19, 8944, 23797, 7, 22, 9, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[0, 19, 8944, 23797, 7, 22, 9, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[8945, 8943, 8944, 8944, 8943, 5951, 5921, 8945] avg=8192.0000 max=8945 max_rank=0 min=5921 min_rank=6 imbalance_ratio=1.091919 max_load_reduction=23793 max_load_reduction_pct=0.726770 moved_to_replicas=80049 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[8945, 8943, 8944, 8944, 8943, 5951, 5921, 8945] avg=8192.0000 max=8945 max_rank=0 min=5921 min_rank=6 imbalance_ratio=1.091919 max_load_reduction=23793 max_load_reduction_pct=0.726770 moved_to_replicas=80049 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[8945, 8943, 8944, 8944, 8943, 5951, 5921, 8945] avg=8192.0000 max=8945 max_rank=0 min=5921 min_rank=6 imbalance_ratio=1.091919 max_load_reduction=23793 max_load_reduction_pct=0.726770 moved_to_replicas=80049 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[24, 29898, 0, 25, 2825, 6, 14, 32744] avg=8192.0000 max=32744 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[0, 19, 8944, 23797, 7, 22, 9, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[8945, 8943, 8944, 8944, 8943, 5951, 5921, 8945] avg=8192.0000 max=8945 max_rank=0 min=5921 min_rank=6 imbalance_ratio=1.091919 max_load_reduction=23793 max_load_reduction_pct=0.726770 moved_to_replicas=80049 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[8945, 8943, 8944, 8944, 8943, 5951, 5921, 8945] avg=8192.0000 max=8945 max_rank=0 min=5921 min_rank=6 imbalance_ratio=1.091919 max_load_reduction=23793 max_load_reduction_pct=0.726770 moved_to_replicas=80049 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[0, 19, 8944, 23797, 7, 22, 9, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[24, 29898, 0, 25, 2825, 6, 14, 32744] avg=8192.0000 max=32744 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7949, 8273, 7942, 8275] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24469 max_load_reduction_pct=0.747282 moved_to_replicas=92264 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[8945, 8943, 8944, 8944, 8943, 5951, 5921, 8945] avg=8192.0000 max=8945 max_rank=0 min=5921 min_rank=6 imbalance_ratio=1.091919 max_load_reduction=23793 max_load_reduction_pct=0.726770 moved_to_replicas=80049 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[24, 29898, 0, 25, 2825, 6, 14, 32744] avg=8192.0000 max=32744 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[24, 29898, 0, 25, 2825, 6, 14, 32744] avg=8192.0000 max=32744 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[8945, 8943, 8944, 8944, 8943, 5951, 5921, 8945] avg=8192.0000 max=8945 max_rank=0 min=5921 min_rank=6 imbalance_ratio=1.091919 max_load_reduction=23793 max_load_reduction_pct=0.726770 moved_to_replicas=80049 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[24, 29898, 0, 25, 2825, 6, 14, 32744] avg=8192.0000 max=32744 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[24, 29898, 0, 25, 2825, 6, 14, 32744] avg=8192.0000 max=32744 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7949, 8273, 7942, 8275] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24469 max_load_reduction_pct=0.747282 moved_to_replicas=92264 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7949, 8273, 7942, 8275] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24469 max_load_reduction_pct=0.747282 moved_to_replicas=92264 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7949, 8273, 7942, 8275] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24469 max_load_reduction_pct=0.747282 moved_to_replicas=92264 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[26, 32738, 32680, 1, 19, 11, 3, 58] avg=8192.0000 max=32738 max_rank=1 min=1 min_rank=3 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[24, 29898, 0, 25, 2825, 6, 14, 32744] avg=8192.0000 max=32744 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7949, 8273, 7942, 8275] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24469 max_load_reduction_pct=0.747282 moved_to_replicas=92264 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7949, 8273, 7942, 8275] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24469 max_load_reduction_pct=0.747282 moved_to_replicas=92264 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[24, 29898, 0, 25, 2825, 6, 14, 32744] avg=8192.0000 max=32744 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997070 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[26, 32738, 32680, 1, 19, 11, 3, 58] avg=8192.0000 max=32738 max_rank=1 min=1 min_rank=3 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[8273, 8274, 8275, 8274, 8274, 7932, 8273, 7961] avg=8192.0000 max=8275 max_rank=2 min=7932 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=97690 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7949, 8273, 7942, 8275] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24469 max_load_reduction_pct=0.747282 moved_to_replicas=92264 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[26, 32738, 32680, 1, 19, 11, 3, 58] avg=8192.0000 max=32738 max_rank=1 min=1 min_rank=3 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[26, 32738, 32680, 1, 19, 11, 3, 58] avg=8192.0000 max=32738 max_rank=1 min=1 min_rank=3 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7949, 8273, 7942, 8275] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24469 max_load_reduction_pct=0.747282 moved_to_replicas=92264 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[8273, 8274, 8275, 8274, 8274, 7932, 8273, 7961] avg=8192.0000 max=8275 max_rank=2 min=7932 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=97690 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[26, 32738, 32680, 1, 19, 11, 3, 58] avg=8192.0000 max=32738 max_rank=1 min=1 min_rank=3 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[26, 32738, 32680, 1, 19, 11, 3, 58] avg=8192.0000 max=32738 max_rank=1 min=1 min_rank=3 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[8273, 8274, 8275, 8274, 8274, 7932, 8273, 7961] avg=8192.0000 max=8275 max_rank=2 min=7932 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=97690 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[11, 32724, 27155, 4, 17, 21, 5594, 10] avg=8192.0000 max=32724 max_rank=1 min=4 min_rank=3 imbalance_ratio=3.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[8273, 8274, 8275, 8274, 8274, 7932, 8273, 7961] avg=8192.0000 max=8275 max_rank=2 min=7932 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=97690 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[26, 32738, 32680, 1, 19, 11, 3, 58] avg=8192.0000 max=32738 max_rank=1 min=1 min_rank=3 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[8273, 8274, 8275, 8274, 8274, 7932, 8273, 7961] avg=8192.0000 max=8275 max_rank=2 min=7932 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=97690 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[8273, 8274, 8275, 8274, 8274, 7932, 8273, 7961] avg=8192.0000 max=8275 max_rank=2 min=7932 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=97690 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[26, 32738, 32680, 1, 19, 11, 3, 58] avg=8192.0000 max=32738 max_rank=1 min=1 min_rank=3 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[11, 32724, 27155, 4, 17, 21, 5594, 10] avg=8192.0000 max=32724 max_rank=1 min=4 min_rank=3 imbalance_ratio=3.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[7927, 8275, 8274, 8274, 8275, 8273, 7965, 8273] avg=8192.0000 max=8275 max_rank=1 min=7927 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24449 max_load_reduction_pct=0.747127 moved_to_replicas=86634 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[8273, 8274, 8275, 8274, 8274, 7932, 8273, 7961] avg=8192.0000 max=8275 max_rank=2 min=7932 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=97690 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[11, 32724, 27155, 4, 17, 21, 5594, 10] avg=8192.0000 max=32724 max_rank=1 min=4 min_rank=3 imbalance_ratio=3.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[11, 32724, 27155, 4, 17, 21, 5594, 10] avg=8192.0000 max=32724 max_rank=1 min=4 min_rank=3 imbalance_ratio=3.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[8273, 8274, 8275, 8274, 8274, 7932, 8273, 7961] avg=8192.0000 max=8275 max_rank=2 min=7932 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=97690 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[7927, 8275, 8274, 8274, 8275, 8273, 7965, 8273] avg=8192.0000 max=8275 max_rank=1 min=7927 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24449 max_load_reduction_pct=0.747127 moved_to_replicas=86634 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[11, 32724, 27155, 4, 17, 21, 5594, 10] avg=8192.0000 max=32724 max_rank=1 min=4 min_rank=3 imbalance_ratio=3.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[11, 32724, 27155, 4, 17, 21, 5594, 10] avg=8192.0000 max=32724 max_rank=1 min=4 min_rank=3 imbalance_ratio=3.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[7927, 8275, 8274, 8274, 8275, 8273, 7965, 8273] avg=8192.0000 max=8275 max_rank=1 min=7927 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24449 max_load_reduction_pct=0.747127 moved_to_replicas=86634 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[7927, 8275, 8274, 8274, 8275, 8273, 7965, 8273] avg=8192.0000 max=8275 max_rank=1 min=7927 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24449 max_load_reduction_pct=0.747127 moved_to_replicas=86634 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[11, 32724, 27155, 4, 17, 21, 5594, 10] avg=8192.0000 max=32724 max_rank=1 min=4 min_rank=3 imbalance_ratio=3.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[7927, 8275, 8274, 8274, 8275, 8273, 7965, 8273] avg=8192.0000 max=8275 max_rank=1 min=7927 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24449 max_load_reduction_pct=0.747127 moved_to_replicas=86634 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[7927, 8275, 8274, 8274, 8275, 8273, 7965, 8273] avg=8192.0000 max=8275 max_rank=1 min=7927 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24449 max_load_reduction_pct=0.747127 moved_to_replicas=86634 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[11, 32724, 27155, 4, 17, 21, 5594, 10] avg=8192.0000 max=32724 max_rank=1 min=4 min_rank=3 imbalance_ratio=3.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[7927, 8275, 8274, 8274, 8275, 8273, 7965, 8273] avg=8192.0000 max=8275 max_rank=1 min=7927 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24449 max_load_reduction_pct=0.747127 moved_to_replicas=86634 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[7927, 8275, 8274, 8274, 8275, 8273, 7965, 8273] avg=8192.0000 max=8275 max_rank=1 min=7927 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24449 max_load_reduction_pct=0.747127 moved_to_replicas=86634 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=32 total_expected_recv=254822 total_dropped_tokens=652 drop_rate_pct=0.255865 records_with_drop_pct=3.125000 peak_rank_overflow=652 total_slack_slots=69504 peak_demand=10892 required_capacity=1.329590 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=32 total_expected_recv=263109 total_dropped_tokens=490 drop_rate_pct=0.186235 records_with_drop_pct=3.125000 peak_rank_overflow=490 total_slack_slots=61056 peak_demand=10730 required_capacity=1.309814 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=32 total_expected_recv=268480 total_dropped_tokens=598 drop_rate_pct=0.222735 records_with_drop_pct=3.125000 peak_rank_overflow=598 total_slack_slots=55552 peak_demand=10838 required_capacity=1.322998 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=32 total_expected_recv=253335 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=70784 peak_demand=8944 required_capacity=1.091797 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=32 total_expected_recv=268693 total_dropped_tokens=653 drop_rate_pct=0.243028 records_with_drop_pct=3.125000 peak_rank_overflow=653 total_slack_slots=55936 peak_demand=10893 required_capacity=1.329712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=32 total_expected_recv=269679 total_dropped_tokens=652 drop_rate_pct=0.241769 records_with_drop_pct=3.125000 peak_rank_overflow=652 total_slack_slots=55040 peak_demand=10892 required_capacity=1.329590 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=32 total_expected_recv=255643 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=68224 peak_demand=8945 required_capacity=1.091919 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=32 total_expected_recv=263391 total_dropped_tokens=653 drop_rate_pct=0.247920 records_with_drop_pct=3.125000 peak_rank_overflow=653 total_slack_slots=60288 peak_demand=10893 required_capacity=1.329712 + [2026-08-30 05:07:59] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 913.5 | throughput per GPU (TFLOP/s/GPU): 365.0 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.116902E+01 | loss scale: 1.0 | grad norm: 87.814 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 10240 | min safe capacity: 1.2500 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=10240 expected_recv=20671 expected_overflow=1211 exact_recv_per_rank=[6202, 8014, 5217, 9220, 9221, 9220, 9221, 9221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=10240 expected_recv=5666 expected_overflow=0 exact_recv_per_rank=[6202, 8014, 5217, 9220, 9221, 9220, 9221, 9221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=10240 expected_recv=12367 expected_overflow=0 exact_recv_per_rank=[6202, 8014, 5217, 9220, 9221, 9220, 9221, 9221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=10240 expected_recv=6358 expected_overflow=0 exact_recv_per_rank=[6202, 8014, 5217, 9220, 9221, 9220, 9221, 9221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=10240 expected_recv=21959 expected_overflow=2499 exact_recv_per_rank=[6202, 8014, 5217, 9220, 9221, 9220, 9221, 9221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=10240 expected_recv=22918 expected_overflow=3457 exact_recv_per_rank=[6202, 8014, 5217, 9220, 9221, 9220, 9221, 9221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=10240 expected_recv=22917 expected_overflow=3456 exact_recv_per_rank=[6202, 8014, 5217, 9220, 9221, 9220, 9221, 9221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=10240 expected_recv=20671 actual_valid_recv=19584 observed_loss=1211 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=10240 expected_recv=5666 actual_valid_recv=5760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=10240 expected_recv=12367 actual_valid_recv=12672 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=10240 expected_recv=18216 expected_overflow=0 exact_recv_per_rank=[6202, 8014, 5217, 9220, 9221, 9220, 9221, 9221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=10240 expected_recv=6358 actual_valid_recv=6528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=10240 expected_recv=21959 actual_valid_recv=19584 observed_loss=2499 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=19584 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=10240 expected_recv=22918 actual_valid_recv=19584 observed_loss=3457 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=5760 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=12672 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=10240 expected_recv=22917 actual_valid_recv=19584 observed_loss=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=6528 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=19584 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=19584 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=10240 expected_recv=18216 actual_valid_recv=18432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=19584 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=18432 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=10240 expected_recv=19045 expected_overflow=0 exact_recv_per_rank=[9506, 8448, 6625, 9506, 9507, 9508, 2929, 9507] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=10240 expected_recv=6636 expected_overflow=0 exact_recv_per_rank=[9506, 8448, 6625, 9506, 9507, 9508, 2929, 9507] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=10240 expected_recv=18708 expected_overflow=20 exact_recv_per_rank=[9506, 8448, 6625, 9506, 9507, 9508, 2929, 9507] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=10240 expected_recv=21352 expected_overflow=1606 exact_recv_per_rank=[9506, 8448, 6625, 9506, 9507, 9508, 2929, 9507] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=10240 expected_recv=19068 expected_overflow=0 exact_recv_per_rank=[9506, 8448, 6625, 9506, 9507, 9508, 2929, 9507] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=10240 expected_recv=19045 actual_valid_recv=19456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=10240 expected_recv=21353 expected_overflow=1606 exact_recv_per_rank=[9506, 8448, 6625, 9506, 9507, 9508, 2929, 9507] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=10240 expected_recv=6636 actual_valid_recv=6912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=10240 expected_recv=18708 actual_valid_recv=18816 observed_loss=20 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=10240 expected_recv=3557 expected_overflow=0 exact_recv_per_rank=[9506, 8448, 6625, 9506, 9507, 9508, 2929, 9507] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=19456 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=10240 expected_recv=19068 actual_valid_recv=19200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=10240 expected_recv=21352 actual_valid_recv=19840 observed_loss=1606 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=10240 expected_recv=21353 actual_valid_recv=19840 observed_loss=1606 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=6912 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=18816 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=10240 expected_recv=3557 actual_valid_recv=3712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=19200 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=19840 slack_slots=640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=19840 slack_slots=640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=3712 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=10240 expected_recv=21353 expected_overflow=1606 exact_recv_per_rank=[9506, 8448, 6625, 9506, 9507, 9508, 2929, 9507] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=10240 expected_recv=21353 actual_valid_recv=19840 observed_loss=1606 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 7944, 8273, 8274, 8275, 7948, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 7944, 8273, 8274, 8275, 7948, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=10240 expected_recv=15890 expected_overflow=0 exact_recv_per_rank=[8273, 7944, 8273, 8274, 8275, 7948, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=10240 expected_recv=16222 expected_overflow=0 exact_recv_per_rank=[8273, 7944, 8273, 8274, 8275, 7948, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=19840 slack_slots=640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=10240 expected_recv=15890 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=10240 expected_recv=16222 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 7944, 8273, 8274, 8275, 7948, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 7944, 8273, 8274, 8275, 7948, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 7944, 8273, 8274, 8275, 7948, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=10240 expected_recv=16221 expected_overflow=0 exact_recv_per_rank=[8273, 7944, 8273, 8274, 8275, 7948, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=10240 expected_recv=16226 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8275, 8273, 7944, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=10240 expected_recv=16221 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=10240 expected_recv=16215 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8275, 8273, 7944, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8275, 8273, 7944, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=10240 expected_recv=16226 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=10240 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8275, 8273, 7944, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=10240 expected_recv=16215 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8275, 8273, 7944, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=10240 expected_recv=16222 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8275, 8273, 7944, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=10240 expected_recv=16217 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8275, 8273, 7944, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=10240 expected_recv=16222 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8275, 8273, 7944, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=10240 expected_recv=16222 expected_overflow=0 exact_recv_per_rank=[8275, 7944, 7946, 8273, 8276, 8275, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=10240 expected_recv=16222 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=10240 expected_recv=16221 expected_overflow=0 exact_recv_per_rank=[8275, 7944, 7946, 8273, 8276, 8275, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8275, 7944, 7946, 8273, 8276, 8275, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 7944, 7946, 8273, 8276, 8275, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=10240 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8275, 7944, 7946, 8273, 8276, 8275, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=10240 expected_recv=16221 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 7944, 7946, 8273, 8276, 8275, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=10240 expected_recv=16545 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 7944, 7946, 8273, 8276, 8275, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8275, 7944, 7946, 8273, 8276, 8275, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 7950, 8275, 8275, 8273, 8274, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=10240 expected_recv=16221 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 7950, 8275, 8275, 8273, 8274, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=10240 expected_recv=15896 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 7950, 8275, 8275, 8273, 8274, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 7950, 8275, 8275, 8273, 8274, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=10240 expected_recv=16214 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 7950, 8275, 8275, 8273, 8274, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=10240 expected_recv=16221 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 7950, 8275, 8275, 8273, 8274, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=10240 expected_recv=15896 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=10240 expected_recv=16214 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 7950, 8275, 8275, 8273, 8274, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 7950, 8275, 8275, 8273, 8274, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7947, 8274, 8276, 8275, 7944, 8274, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7947, 8274, 8276, 8275, 7944, 8274, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7947, 8274, 8276, 8275, 7944, 8274, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7947, 8274, 8276, 8275, 7944, 8274, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7947, 8274, 8276, 8275, 7944, 8274, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=10240 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[7947, 8274, 8276, 8275, 7944, 8274, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=10240 expected_recv=15902 expected_overflow=0 exact_recv_per_rank=[7947, 8274, 8276, 8275, 7944, 8274, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=10240 expected_recv=16545 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=10240 expected_recv=15881 expected_overflow=0 exact_recv_per_rank=[7947, 8274, 8276, 8275, 7944, 8274, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8273, 8276, 8274, 7944, 7946, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=10240 expected_recv=15902 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=10240 expected_recv=15881 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 8273, 8276, 8274, 7944, 7946, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8273, 8276, 8274, 7944, 7946, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=10240 expected_recv=15892 expected_overflow=0 exact_recv_per_rank=[8276, 8273, 8276, 8274, 7944, 7946, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 8273, 8276, 8274, 7944, 7946, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=10240 expected_recv=15892 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8273, 8276, 8274, 7944, 7946, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8273, 8276, 8274, 7944, 7946, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=10240 expected_recv=15889 expected_overflow=0 exact_recv_per_rank=[8276, 8273, 8276, 8274, 7944, 7946, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=10240 expected_recv=15889 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=10240 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=10240 expected_recv=16215 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=10240 expected_recv=16220 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=10240 expected_recv=16224 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=10240 expected_recv=16215 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=10240 expected_recv=16225 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8276, 8272, 7944, 8273, 7945, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=10240 expected_recv=16224 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=10240 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=10240 expected_recv=16225 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=10240 expected_recv=16217 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8276, 8272, 7944, 8273, 7945, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8276, 8272, 7944, 8273, 7945, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=10240 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8276, 8272, 7944, 8273, 7945, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=10240 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8276, 8272, 7944, 8273, 7945, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=10240 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8276, 8272, 7944, 8273, 7945, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8272, 8275, 8273, 7946, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=10240 expected_recv=16545 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=10240 expected_recv=16216 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=10240 expected_recv=16220 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=10240 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8276, 8272, 7944, 8273, 7945, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8276, 8272, 7944, 8273, 7945, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=10240 expected_recv=16217 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8272, 8275, 8273, 7946, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8272, 8275, 8273, 7946, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=10240 expected_recv=16225 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8272, 8275, 8273, 7946, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8272, 8275, 8273, 7946, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8272, 8275, 8273, 7946, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 7948, 8276, 8276, 8272, 7944, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=10240 expected_recv=16225 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=10240 expected_recv=16214 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8272, 8275, 8273, 7946, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8272, 8275, 8273, 7946, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=10240 expected_recv=16214 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=10240 expected_recv=16222 expected_overflow=0 exact_recv_per_rank=[8276, 7948, 8276, 8276, 8272, 7944, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=10240 expected_recv=16228 expected_overflow=0 exact_recv_per_rank=[8276, 7948, 8276, 8276, 8272, 7944, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=10240 expected_recv=16222 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=10240 expected_recv=15881 expected_overflow=0 exact_recv_per_rank=[8276, 7948, 8276, 8276, 8272, 7944, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 7948, 8276, 8276, 8272, 7944, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=10240 expected_recv=16228 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 7948, 8276, 8276, 8272, 7944, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=10240 expected_recv=15881 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=10240 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 7948, 8276, 8276, 8272, 7944, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=10240 expected_recv=16217 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 7948, 8276, 8276, 8272, 7944, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=10240 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=10240 expected_recv=15883 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=10240 expected_recv=15900 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=10240 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=10240 expected_recv=16230 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=10240 expected_recv=15883 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=10240 expected_recv=15900 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=10240 expected_recv=16230 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=10240 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=10240 expected_recv=16205 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=10240 expected_recv=15883 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=10240 expected_recv=16220 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=10240 expected_recv=16205 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=10240 expected_recv=15883 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=10240 expected_recv=16230 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=10240 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=10240 expected_recv=16230 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=10240 expected_recv=16212 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=10240 expected_recv=16545 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=10240 expected_recv=16212 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=10240 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=10240 expected_recv=16226 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=10240 expected_recv=16220 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=10240 expected_recv=16226 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=10240 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[65, 24, 507, 11681, 6760, 15357, 17211, 13931] avg=8192.0000 max=17211 max_rank=6 min=24 min_rank=1 imbalance_ratio=2.100952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=10240 expected_recv=16220 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[65, 24, 507, 11681, 6760, 15357, 17211, 13931] avg=8192.0000 max=17211 max_rank=6 min=24 min_rank=1 imbalance_ratio=2.100952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[6202, 8014, 5217, 9220, 9221, 9220, 9221, 9221] avg=8192.0000 max=9221 max_rank=4 min=5217 min_rank=2 imbalance_ratio=1.125610 max_load_reduction=7990 max_load_reduction_pct=0.464238 moved_to_replicas=25633 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[65, 24, 507, 11681, 6760, 15357, 17211, 13931] avg=8192.0000 max=17211 max_rank=6 min=24 min_rank=1 imbalance_ratio=2.100952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[65, 24, 507, 11681, 6760, 15357, 17211, 13931] avg=8192.0000 max=17211 max_rank=6 min=24 min_rank=1 imbalance_ratio=2.100952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[65, 24, 507, 11681, 6760, 15357, 17211, 13931] avg=8192.0000 max=17211 max_rank=6 min=24 min_rank=1 imbalance_ratio=2.100952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[65, 24, 507, 11681, 6760, 15357, 17211, 13931] avg=8192.0000 max=17211 max_rank=6 min=24 min_rank=1 imbalance_ratio=2.100952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[6202, 8014, 5217, 9220, 9221, 9220, 9221, 9221] avg=8192.0000 max=9221 max_rank=4 min=5217 min_rank=2 imbalance_ratio=1.125610 max_load_reduction=7990 max_load_reduction_pct=0.464238 moved_to_replicas=25633 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[65, 24, 507, 11681, 6760, 15357, 17211, 13931] avg=8192.0000 max=17211 max_rank=6 min=24 min_rank=1 imbalance_ratio=2.100952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[16123, 4, 8, 2, 17951, 11690, 747, 19011] avg=8192.0000 max=19011 max_rank=7 min=2 min_rank=3 imbalance_ratio=2.320679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[65, 24, 507, 11681, 6760, 15357, 17211, 13931] avg=8192.0000 max=17211 max_rank=6 min=24 min_rank=1 imbalance_ratio=2.100952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[6202, 8014, 5217, 9220, 9221, 9220, 9221, 9221] avg=8192.0000 max=9221 max_rank=4 min=5217 min_rank=2 imbalance_ratio=1.125610 max_load_reduction=7990 max_load_reduction_pct=0.464238 moved_to_replicas=25633 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[6202, 8014, 5217, 9220, 9221, 9220, 9221, 9221] avg=8192.0000 max=9221 max_rank=4 min=5217 min_rank=2 imbalance_ratio=1.125610 max_load_reduction=7990 max_load_reduction_pct=0.464238 moved_to_replicas=25633 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[6202, 8014, 5217, 9220, 9221, 9220, 9221, 9221] avg=8192.0000 max=9221 max_rank=4 min=5217 min_rank=2 imbalance_ratio=1.125610 max_load_reduction=7990 max_load_reduction_pct=0.464238 moved_to_replicas=25633 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[6202, 8014, 5217, 9220, 9221, 9220, 9221, 9221] avg=8192.0000 max=9221 max_rank=4 min=5217 min_rank=2 imbalance_ratio=1.125610 max_load_reduction=7990 max_load_reduction_pct=0.464238 moved_to_replicas=25633 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[6202, 8014, 5217, 9220, 9221, 9220, 9221, 9221] avg=8192.0000 max=9221 max_rank=4 min=5217 min_rank=2 imbalance_ratio=1.125610 max_load_reduction=7990 max_load_reduction_pct=0.464238 moved_to_replicas=25633 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[9506, 8448, 6625, 9506, 9507, 9508, 2929, 9507] avg=8192.0000 max=9508 max_rank=5 min=2929 min_rank=6 imbalance_ratio=1.160645 max_load_reduction=9503 max_load_reduction_pct=0.499868 moved_to_replicas=46539 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[16123, 4, 8, 2, 17951, 11690, 747, 19011] avg=8192.0000 max=19011 max_rank=7 min=2 min_rank=3 imbalance_ratio=2.320679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[6202, 8014, 5217, 9220, 9221, 9220, 9221, 9221] avg=8192.0000 max=9221 max_rank=4 min=5217 min_rank=2 imbalance_ratio=1.125610 max_load_reduction=7990 max_load_reduction_pct=0.464238 moved_to_replicas=25633 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[16123, 4, 8, 2, 17951, 11690, 747, 19011] avg=8192.0000 max=19011 max_rank=7 min=2 min_rank=3 imbalance_ratio=2.320679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[16123, 4, 8, 2, 17951, 11690, 747, 19011] avg=8192.0000 max=19011 max_rank=7 min=2 min_rank=3 imbalance_ratio=2.320679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[16123, 4, 8, 2, 17951, 11690, 747, 19011] avg=8192.0000 max=19011 max_rank=7 min=2 min_rank=3 imbalance_ratio=2.320679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[9506, 8448, 6625, 9506, 9507, 9508, 2929, 9507] avg=8192.0000 max=9508 max_rank=5 min=2929 min_rank=6 imbalance_ratio=1.160645 max_load_reduction=9503 max_load_reduction_pct=0.499868 moved_to_replicas=46539 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[16123, 4, 8, 2, 17951, 11690, 747, 19011] avg=8192.0000 max=19011 max_rank=7 min=2 min_rank=3 imbalance_ratio=2.320679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[16123, 4, 8, 2, 17951, 11690, 747, 19011] avg=8192.0000 max=19011 max_rank=7 min=2 min_rank=3 imbalance_ratio=2.320679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[1, 733, 19, 32009, 5, 4, 32765, 0] avg=8192.0000 max=32765 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[9506, 8448, 6625, 9506, 9507, 9508, 2929, 9507] avg=8192.0000 max=9508 max_rank=5 min=2929 min_rank=6 imbalance_ratio=1.160645 max_load_reduction=9503 max_load_reduction_pct=0.499868 moved_to_replicas=46539 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[16123, 4, 8, 2, 17951, 11690, 747, 19011] avg=8192.0000 max=19011 max_rank=7 min=2 min_rank=3 imbalance_ratio=2.320679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[9506, 8448, 6625, 9506, 9507, 9508, 2929, 9507] avg=8192.0000 max=9508 max_rank=5 min=2929 min_rank=6 imbalance_ratio=1.160645 max_load_reduction=9503 max_load_reduction_pct=0.499868 moved_to_replicas=46539 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[9506, 8448, 6625, 9506, 9507, 9508, 2929, 9507] avg=8192.0000 max=9508 max_rank=5 min=2929 min_rank=6 imbalance_ratio=1.160645 max_load_reduction=9503 max_load_reduction_pct=0.499868 moved_to_replicas=46539 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[9506, 8448, 6625, 9506, 9507, 9508, 2929, 9507] avg=8192.0000 max=9508 max_rank=5 min=2929 min_rank=6 imbalance_ratio=1.160645 max_load_reduction=9503 max_load_reduction_pct=0.499868 moved_to_replicas=46539 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[9506, 8448, 6625, 9506, 9507, 9508, 2929, 9507] avg=8192.0000 max=9508 max_rank=5 min=2929 min_rank=6 imbalance_ratio=1.160645 max_load_reduction=9503 max_load_reduction_pct=0.499868 moved_to_replicas=46539 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[8273, 7944, 8273, 8274, 8275, 7948, 8275, 8274] avg=8192.0000 max=8275 max_rank=4 min=7944 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=96696 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[1, 733, 19, 32009, 5, 4, 32765, 0] avg=8192.0000 max=32765 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[9506, 8448, 6625, 9506, 9507, 9508, 2929, 9507] avg=8192.0000 max=9508 max_rank=5 min=2929 min_rank=6 imbalance_ratio=1.160645 max_load_reduction=9503 max_load_reduction_pct=0.499868 moved_to_replicas=46539 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[1, 733, 19, 32009, 5, 4, 32765, 0] avg=8192.0000 max=32765 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[1, 733, 19, 32009, 5, 4, 32765, 0] avg=8192.0000 max=32765 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[1, 733, 19, 32009, 5, 4, 32765, 0] avg=8192.0000 max=32765 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[8273, 7944, 8273, 8274, 8275, 7948, 8275, 8274] avg=8192.0000 max=8275 max_rank=4 min=7944 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=96696 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[1, 733, 19, 32009, 5, 4, 32765, 0] avg=8192.0000 max=32765 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[1, 733, 19, 32009, 5, 4, 32765, 0] avg=8192.0000 max=32765 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[32755, 0, 0, 2, 2, 0, 10, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[8273, 7944, 8273, 8274, 8275, 7948, 8275, 8274] avg=8192.0000 max=8275 max_rank=4 min=7944 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=96696 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[1, 733, 19, 32009, 5, 4, 32765, 0] avg=8192.0000 max=32765 max_rank=6 min=0 min_rank=7 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[8273, 7944, 8273, 8274, 8275, 7948, 8275, 8274] avg=8192.0000 max=8275 max_rank=4 min=7944 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=96696 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[8273, 7944, 8273, 8274, 8275, 7948, 8275, 8274] avg=8192.0000 max=8275 max_rank=4 min=7944 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=96696 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[8273, 7944, 8273, 8274, 8275, 7948, 8275, 8274] avg=8192.0000 max=8275 max_rank=4 min=7944 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=96696 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[8273, 7944, 8273, 8274, 8275, 7948, 8275, 8274] avg=8192.0000 max=8275 max_rank=4 min=7944 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=96696 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[8274, 8276, 8275, 8275, 8273, 7944, 7947, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[32755, 0, 0, 2, 2, 0, 10, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[8273, 7944, 8273, 8274, 8275, 7948, 8275, 8274] avg=8192.0000 max=8275 max_rank=4 min=7944 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=96696 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[32755, 0, 0, 2, 2, 0, 10, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[8274, 8276, 8275, 8275, 8273, 7944, 7947, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[32755, 0, 0, 2, 2, 0, 10, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[32755, 0, 0, 2, 2, 0, 10, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[32755, 0, 0, 2, 2, 0, 10, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[32755, 0, 0, 2, 2, 0, 10, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[32763, 1, 3, 2, 0, 0, 1, 32766] avg=8192.0000 max=32766 max_rank=7 min=0 min_rank=4 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[8274, 8276, 8275, 8275, 8273, 7944, 7947, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[32755, 0, 0, 2, 2, 0, 10, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[8274, 8276, 8275, 8275, 8273, 7944, 7947, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[8274, 8276, 8275, 8275, 8273, 7944, 7947, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[8275, 7944, 7946, 8273, 8276, 8275, 8275, 8272] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[8274, 8276, 8275, 8275, 8273, 7944, 7947, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[8274, 8276, 8275, 8275, 8273, 7944, 7947, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[32763, 1, 3, 2, 0, 0, 1, 32766] avg=8192.0000 max=32766 max_rank=7 min=0 min_rank=4 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[8274, 8276, 8275, 8275, 8273, 7944, 7947, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[32763, 1, 3, 2, 0, 0, 1, 32766] avg=8192.0000 max=32766 max_rank=7 min=0 min_rank=4 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[8275, 7944, 7946, 8273, 8276, 8275, 8275, 8272] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[32763, 1, 3, 2, 0, 0, 1, 32766] avg=8192.0000 max=32766 max_rank=7 min=0 min_rank=4 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[32763, 1, 3, 2, 0, 0, 1, 32766] avg=8192.0000 max=32766 max_rank=7 min=0 min_rank=4 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[30, 0, 107, 5, 0, 32630, 32764, 0] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[32763, 1, 3, 2, 0, 0, 1, 32766] avg=8192.0000 max=32766 max_rank=7 min=0 min_rank=4 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[32763, 1, 3, 2, 0, 0, 1, 32766] avg=8192.0000 max=32766 max_rank=7 min=0 min_rank=4 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[8275, 7944, 7946, 8273, 8276, 8275, 8275, 8272] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[32763, 1, 3, 2, 0, 0, 1, 32766] avg=8192.0000 max=32766 max_rank=7 min=0 min_rank=4 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[8275, 7944, 7946, 8273, 8276, 8275, 8275, 8272] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[8275, 7944, 7946, 8273, 8276, 8275, 8275, 8272] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[8274, 8275, 7950, 8275, 8275, 8273, 8274, 7940] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97582 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[8275, 7944, 7946, 8273, 8276, 8275, 8275, 8272] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[30, 0, 107, 5, 0, 32630, 32764, 0] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[8275, 7944, 7946, 8273, 8276, 8275, 8275, 8272] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[8275, 7944, 7946, 8273, 8276, 8275, 8275, 8272] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[30, 0, 107, 5, 0, 32630, 32764, 0] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[8274, 8275, 7950, 8275, 8275, 8273, 8274, 7940] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97582 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[30, 0, 107, 5, 0, 32630, 32764, 0] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[30, 0, 107, 5, 0, 32630, 32764, 0] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[4469, 9, 0, 0, 0, 28291, 32767, 0] avg=8192.0000 max=32767 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[30, 0, 107, 5, 0, 32630, 32764, 0] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[8274, 8275, 7950, 8275, 8275, 8273, 8274, 7940] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97582 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[30, 0, 107, 5, 0, 32630, 32764, 0] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[30, 0, 107, 5, 0, 32630, 32764, 0] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[8274, 8275, 7950, 8275, 8275, 8273, 8274, 7940] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97582 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[8274, 8275, 7950, 8275, 8275, 8273, 8274, 7940] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97582 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[7947, 8274, 8276, 8275, 7944, 8274, 8272, 8274] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=89844 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[8274, 8275, 7950, 8275, 8275, 8273, 8274, 7940] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97582 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[4469, 9, 0, 0, 0, 28291, 32767, 0] avg=8192.0000 max=32767 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[8274, 8275, 7950, 8275, 8275, 8273, 8274, 7940] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97582 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[8274, 8275, 7950, 8275, 8275, 8273, 8274, 7940] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97582 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[4469, 9, 0, 0, 0, 28291, 32767, 0] avg=8192.0000 max=32767 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[7947, 8274, 8276, 8275, 7944, 8274, 8272, 8274] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=89844 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[4469, 9, 0, 0, 0, 28291, 32767, 0] avg=8192.0000 max=32767 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[4469, 9, 0, 0, 0, 28291, 32767, 0] avg=8192.0000 max=32767 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[0, 31657, 0, 4, 0, 1107, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[4469, 9, 0, 0, 0, 28291, 32767, 0] avg=8192.0000 max=32767 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[7947, 8274, 8276, 8275, 7944, 8274, 8272, 8274] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=89844 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[4469, 9, 0, 0, 0, 28291, 32767, 0] avg=8192.0000 max=32767 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[4469, 9, 0, 0, 0, 28291, 32767, 0] avg=8192.0000 max=32767 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[7947, 8274, 8276, 8275, 7944, 8274, 8272, 8274] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=89844 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[7947, 8274, 8276, 8275, 7944, 8274, 8272, 8274] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=89844 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[8276, 8273, 8276, 8274, 7944, 7946, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=95965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[7947, 8274, 8276, 8275, 7944, 8274, 8272, 8274] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=89844 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[0, 31657, 0, 4, 0, 1107, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[7947, 8274, 8276, 8275, 7944, 8274, 8272, 8274] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=89844 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[7947, 8274, 8276, 8275, 7944, 8274, 8272, 8274] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=89844 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[0, 31657, 0, 4, 0, 1107, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[0, 31657, 0, 4, 0, 1107, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[8276, 8273, 8276, 8274, 7944, 7946, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=95965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[0, 31657, 0, 4, 0, 1107, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[32767, 1, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[8276, 8273, 8276, 8274, 7944, 7946, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=95965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[0, 31657, 0, 4, 0, 1107, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[0, 31657, 0, 4, 0, 1107, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[0, 31657, 0, 4, 0, 1107, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[8276, 8273, 8276, 8274, 7944, 7946, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=95965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[8276, 8273, 8276, 8274, 7944, 7946, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=95965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[8276, 8273, 8276, 8274, 7944, 7946, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=95965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[8276, 8273, 8276, 8274, 7944, 7946, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=95965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[32767, 1, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[8276, 8273, 8276, 8274, 7944, 7946, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=95965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[32767, 1, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[32767, 1, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[0, 32766, 32768, 0, 0, 1, 1, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[32767, 1, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[32767, 1, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[32767, 1, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[32767, 1, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[8276, 8275, 8276, 8272, 7944, 8273, 7945, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[0, 32766, 32768, 0, 0, 1, 1, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[0, 32766, 32768, 0, 0, 1, 1, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[0, 32766, 32768, 0, 0, 1, 1, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[0, 0, 0, 32768, 0, 0, 7, 32761] avg=8192.0000 max=32768 max_rank=3 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[8276, 8275, 8276, 8272, 7944, 8273, 7945, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[0, 32766, 32768, 0, 0, 1, 1, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[8276, 8275, 8276, 8272, 7944, 8273, 7945, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[0, 32766, 32768, 0, 0, 1, 1, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[0, 32766, 32768, 0, 0, 1, 1, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[8276, 8275, 8276, 8272, 7944, 8273, 7945, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[0, 32766, 32768, 0, 0, 1, 1, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[8276, 8276, 7944, 8272, 8275, 8273, 7946, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[8276, 8275, 8276, 8272, 7944, 8273, 7945, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[8276, 8275, 8276, 8272, 7944, 8273, 7945, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[0, 0, 0, 32768, 0, 0, 7, 32761] avg=8192.0000 max=32768 max_rank=3 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[8276, 8275, 8276, 8272, 7944, 8273, 7945, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[8276, 8275, 8276, 8272, 7944, 8273, 7945, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[0, 0, 0, 32768, 0, 0, 7, 32761] avg=8192.0000 max=32768 max_rank=3 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[0, 0, 0, 32768, 0, 0, 7, 32761] avg=8192.0000 max=32768 max_rank=3 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[32752, 16, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[8276, 8276, 7944, 8272, 8275, 8273, 7946, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[8276, 8276, 7944, 8272, 8275, 8273, 7946, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[0, 0, 0, 32768, 0, 0, 7, 32761] avg=8192.0000 max=32768 max_rank=3 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[0, 0, 0, 32768, 0, 0, 7, 32761] avg=8192.0000 max=32768 max_rank=3 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[8276, 8276, 7944, 8272, 8275, 8273, 7946, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[0, 0, 0, 32768, 0, 0, 7, 32761] avg=8192.0000 max=32768 max_rank=3 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[0, 0, 0, 32768, 0, 0, 7, 32761] avg=8192.0000 max=32768 max_rank=3 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[8276, 7948, 8276, 8276, 8272, 7944, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[8276, 8276, 7944, 8272, 8275, 8273, 7946, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[8276, 8276, 7944, 8272, 8275, 8273, 7946, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[32752, 16, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[8276, 8276, 7944, 8272, 8275, 8273, 7946, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[32752, 16, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[8276, 8276, 7944, 8272, 8275, 8273, 7946, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[32752, 16, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[0, 0, 32768, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[8276, 7948, 8276, 8276, 8272, 7944, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[8276, 7948, 8276, 8276, 8272, 7944, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[8276, 7948, 8276, 8276, 8272, 7944, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[32752, 16, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97927 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[32752, 16, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[32752, 16, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[32752, 16, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[8276, 7948, 8276, 8276, 8272, 7944, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[0, 0, 32768, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[0, 0, 32768, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[0, 0, 32768, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[8276, 7948, 8276, 8276, 8272, 7944, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[8276, 7948, 8276, 8276, 8272, 7944, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[8276, 7948, 8276, 8276, 8272, 7944, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[0, 0, 1, 0, 32767, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97927 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97927 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97927 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[0, 0, 32768, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[0, 0, 32768, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[0, 0, 32768, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[0, 0, 32768, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[0, 0, 1, 0, 32767, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[0, 0, 1, 0, 32767, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97927 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[0, 0, 1, 0, 32767, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97927 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97927 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97927 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[0, 32768, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[0, 0, 1, 0, 32767, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[0, 0, 1, 0, 32767, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[0, 0, 1, 0, 32767, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[0, 0, 1, 0, 32767, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[0, 32768, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[0, 32768, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[0, 32768, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[32767, 0, 1, 0, 0, 0, 32768, 0] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[0, 32768, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[0, 32768, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[0, 32768, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[0, 32768, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[32767, 0, 1, 0, 0, 0, 32768, 0] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[32767, 0, 1, 0, 0, 0, 32768, 0] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[32767, 0, 1, 0, 0, 0, 32768, 0] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[32767, 0, 1, 0, 0, 0, 32768, 0] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[32767, 0, 1, 0, 0, 0, 32768, 0] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[32767, 0, 1, 0, 0, 0, 32768, 0] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[32767, 0, 1, 0, 0, 0, 32768, 0] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=32 total_expected_recv=268444 total_dropped_tokens=1211 drop_rate_pct=0.451118 records_with_drop_pct=3.125000 peak_rank_overflow=1211 total_slack_slots=56320 peak_demand=11451 required_capacity=1.397827 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=32 total_expected_recv=274607 total_dropped_tokens=5063 drop_rate_pct=1.843726 records_with_drop_pct=6.250000 peak_rank_overflow=3457 total_slack_slots=55424 peak_demand=13697 required_capacity=1.671997 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=32 total_expected_recv=267620 total_dropped_tokens=1606 drop_rate_pct=0.600105 records_with_drop_pct=3.125000 peak_rank_overflow=1606 total_slack_slots=58240 peak_demand=11846 required_capacity=1.446045 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=32 total_expected_recv=260458 total_dropped_tokens=20 drop_rate_pct=0.007679 records_with_drop_pct=3.125000 peak_rank_overflow=20 total_slack_slots=64256 peak_demand=10260 required_capacity=1.252441 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=32 total_expected_recv=256491 total_dropped_tokens=3456 drop_rate_pct=1.347416 records_with_drop_pct=3.125000 peak_rank_overflow=3456 total_slack_slots=71296 peak_demand=13696 required_capacity=1.671875 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=32 total_expected_recv=258766 total_dropped_tokens=1606 drop_rate_pct=0.620638 records_with_drop_pct=3.125000 peak_rank_overflow=1606 total_slack_slots=67840 peak_demand=11846 required_capacity=1.446045 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=32 total_expected_recv=269410 total_dropped_tokens=2499 drop_rate_pct=0.927582 records_with_drop_pct=3.125000 peak_rank_overflow=2499 total_slack_slots=57600 peak_demand=12739 required_capacity=1.555054 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=32 total_expected_recv=241356 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=82688 peak_demand=8276 required_capacity=1.010254 + [2026-08-30 05:08:00] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 1283.4 | throughput per GPU (TFLOP/s/GPU): 259.8 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.018590E+01 | loss scale: 1.0 | grad norm: 15.618 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 10240 | min safe capacity: 1.2500 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=10240 expected_recv=17766 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 4587, 8930, 7937, 8789, 8931, 8502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=10240 expected_recv=17860 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 4587, 8930, 7937, 8789, 8931, 8502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=10240 expected_recv=17860 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 4587, 8930, 7937, 8789, 8931, 8502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=10240 expected_recv=8273 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 4587, 8930, 7937, 8789, 8931, 8502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=10240 expected_recv=17612 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 4587, 8930, 7937, 8789, 8931, 8502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=10240 expected_recv=17432 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 4587, 8930, 7937, 8789, 8931, 8502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=10240 expected_recv=17861 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 4587, 8930, 7937, 8789, 8931, 8502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=10240 expected_recv=17766 actual_valid_recv=18048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=10240 expected_recv=17860 actual_valid_recv=18176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=10240 expected_recv=17860 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=10240 expected_recv=8273 actual_valid_recv=8576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=10240 expected_recv=16408 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 4587, 8930, 7937, 8789, 8931, 8502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=10240 expected_recv=17612 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=18048 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=18176 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17920 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=10240 expected_recv=17432 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=10240 expected_recv=17861 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=8576 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17664 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17664 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=10240 expected_recv=16408 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17920 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7941, 8273, 8275, 8275, 7951, 8274, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=10240 expected_recv=16214 expected_overflow=0 exact_recv_per_rank=[7941, 8273, 8275, 8275, 7951, 8274, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=10240 expected_recv=16213 expected_overflow=0 exact_recv_per_rank=[7941, 8273, 8275, 8275, 7951, 8274, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=10240 expected_recv=16214 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=10240 expected_recv=16213 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7941, 8273, 8275, 8275, 7951, 8274, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[7941, 8273, 8275, 8275, 7951, 8274, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7941, 8273, 8275, 8275, 7951, 8274, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7941, 8273, 8275, 8275, 7951, 8274, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=10240 expected_recv=15902 expected_overflow=0 exact_recv_per_rank=[7941, 8273, 8275, 8275, 7951, 8274, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=10240 expected_recv=15902 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7937, 8275, 7955, 8275, 8273, 8274, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=10240 expected_recv=16212 expected_overflow=0 exact_recv_per_rank=[7937, 8275, 7955, 8275, 8273, 8274, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7937, 8275, 7955, 8275, 8273, 8274, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=10240 expected_recv=16212 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7937, 8275, 7955, 8275, 8273, 8274, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7937, 8275, 7955, 8275, 8273, 8274, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=10240 expected_recv=15909 expected_overflow=0 exact_recv_per_rank=[7937, 8275, 7955, 8275, 8273, 8274, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7937, 8275, 7955, 8275, 8273, 8274, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=10240 expected_recv=15909 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=10240 expected_recv=16210 expected_overflow=0 exact_recv_per_rank=[7937, 8275, 7955, 8275, 8273, 8274, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=10240 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7948, 7945, 8273, 8275, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=10240 expected_recv=16210 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7948, 7945, 8273, 8275, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=10240 expected_recv=15888 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7948, 7945, 8273, 8275, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7948, 7945, 8273, 8275, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=10240 expected_recv=16221 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7948, 7945, 8273, 8275, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=10240 expected_recv=16223 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7948, 7945, 8273, 8275, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7948, 7945, 8273, 8275, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7948, 7945, 8273, 8275, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=10240 expected_recv=16221 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=10240 expected_recv=16223 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=10240 expected_recv=23137 expected_overflow=2657 exact_recv_per_rank=[11211, 11211, 10341, 11212, 4, 4, 11211, 10342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=10240 expected_recv=23137 actual_valid_recv=20480 observed_loss=2657 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=10240 expected_recv=11213 expected_overflow=971 exact_recv_per_rank=[11211, 11211, 10341, 11212, 4, 4, 11211, 10342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=10240 expected_recv=23134 expected_overflow=2654 exact_recv_per_rank=[11211, 11211, 10341, 11212, 4, 4, 11211, 10342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=20480 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=10240 expected_recv=11213 actual_valid_recv=10368 observed_loss=971 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=10240 expected_recv=11930 expected_overflow=1686 exact_recv_per_rank=[11211, 11211, 10341, 11212, 4, 4, 11211, 10342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=10240 expected_recv=23134 actual_valid_recv=20480 observed_loss=2654 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=10368 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=10240 expected_recv=19258 expected_overflow=102 exact_recv_per_rank=[11211, 11211, 10341, 11212, 4, 4, 11211, 10342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=20480 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=10240 expected_recv=11930 actual_valid_recv=10368 observed_loss=1686 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=10240 expected_recv=22266 expected_overflow=1786 exact_recv_per_rank=[11211, 11211, 10341, 11212, 4, 4, 11211, 10342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=10240 expected_recv=11215 expected_overflow=971 exact_recv_per_rank=[11211, 11211, 10341, 11212, 4, 4, 11211, 10342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=10240 expected_recv=19258 actual_valid_recv=19328 observed_loss=102 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=10368 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=10240 expected_recv=8919 expected_overflow=0 exact_recv_per_rank=[11211, 11211, 10341, 11212, 4, 4, 11211, 10342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=10240 expected_recv=22266 actual_valid_recv=20480 observed_loss=1786 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=19328 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=10240 expected_recv=11215 actual_valid_recv=10368 observed_loss=971 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 7948, 8276, 8275, 8274, 7941, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=20480 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=10240 expected_recv=8919 actual_valid_recv=9216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=10368 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=10240 expected_recv=15899 expected_overflow=0 exact_recv_per_rank=[8275, 7948, 8276, 8275, 8274, 7941, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 7948, 8276, 8275, 8274, 7941, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=9216 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=10240 expected_recv=15881 expected_overflow=0 exact_recv_per_rank=[8275, 7948, 8276, 8275, 8274, 7941, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=10240 expected_recv=15899 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 7948, 8276, 8275, 8274, 7941, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=10240 expected_recv=15881 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 7948, 8276, 8275, 8274, 7941, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 7948, 8276, 8275, 8274, 7941, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 7948, 8276, 8275, 8274, 7941, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=10240 expected_recv=16225 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 7941, 8276, 8272, 8275, 8274, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 7941, 8276, 8272, 8275, 8274, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=10240 expected_recv=16225 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 7941, 8276, 8272, 8275, 8274, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 7941, 8276, 8272, 8275, 8274, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=10240 expected_recv=16221 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 7941, 8276, 8272, 8275, 8274, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=10240 expected_recv=16221 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=10240 expected_recv=15881 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 7941, 8276, 8272, 8275, 8274, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 7941, 8276, 8272, 8275, 8274, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 7941, 8276, 8272, 8275, 8274, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=10240 expected_recv=15881 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8272, 8274, 7952, 8273, 7938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8272, 8274, 7952, 8273, 7938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8272, 8274, 7952, 8273, 7938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=10240 expected_recv=15907 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8272, 8274, 7952, 8273, 7938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=10240 expected_recv=15874 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8272, 8274, 7952, 8273, 7938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=10240 expected_recv=15907 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=10240 expected_recv=15874 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8272, 8274, 7952, 8273, 7938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8272, 8274, 7952, 8273, 7938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8272, 8274, 7952, 8273, 7938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=10240 expected_recv=15899 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=10240 expected_recv=16213 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=10240 expected_recv=16214 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=10240 expected_recv=15899 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=10240 expected_recv=16224 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7952, 8275, 8272, 8274, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=10240 expected_recv=16213 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=10240 expected_recv=16214 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=10240 expected_recv=16224 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7952, 8275, 8272, 8274, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7952, 8275, 8272, 8274, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=10240 expected_recv=16214 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7952, 8275, 8272, 8274, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=10240 expected_recv=16212 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7952, 8275, 8272, 8274, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=10240 expected_recv=16214 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=10240 expected_recv=16212 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7952, 8275, 8272, 8274, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=10240 expected_recv=16223 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7952, 8275, 8272, 8274, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7952, 8275, 8272, 8274, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=10240 expected_recv=15879 expected_overflow=0 exact_recv_per_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=10240 expected_recv=16223 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=10240 expected_recv=15879 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=10240 expected_recv=15904 expected_overflow=0 exact_recv_per_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=10240 expected_recv=15904 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8274, 8274, 7937, 8273, 8274, 7956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8274, 8274, 7937, 8273, 8274, 7956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8274, 8274, 7937, 8273, 8274, 7956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8274, 8274, 7937, 8273, 8274, 7956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=10240 expected_recv=15911 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8274, 8274, 7937, 8273, 8274, 7956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=10240 expected_recv=15911 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=10240 expected_recv=15873 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8274, 8274, 7937, 8273, 8274, 7956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8274, 8274, 7937, 8273, 8274, 7956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8274, 8274, 7937, 8273, 8274, 7956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=10240 expected_recv=15873 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=10240 expected_recv=16224 expected_overflow=0 exact_recv_per_rank=[8275, 7941, 8274, 7950, 8274, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=10240 expected_recv=16224 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 7941, 8274, 7950, 8274, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=10240 expected_recv=16226 expected_overflow=0 exact_recv_per_rank=[8275, 7941, 8274, 7950, 8274, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=10240 expected_recv=15882 expected_overflow=0 exact_recv_per_rank=[8275, 7941, 8274, 7950, 8274, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 7941, 8274, 7950, 8274, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=10240 expected_recv=16226 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=10240 expected_recv=15882 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 7941, 8274, 7950, 8274, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 7941, 8274, 7950, 8274, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 7941, 8274, 7950, 8274, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[7952, 8273, 8275, 8273, 8275, 8276, 8274, 7938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=10240 expected_recv=15914 expected_overflow=0 exact_recv_per_rank=[7952, 8273, 8275, 8273, 8275, 8276, 8274, 7938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=10240 expected_recv=15914 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7952, 8273, 8275, 8273, 8275, 8276, 8274, 7938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7952, 8273, 8275, 8273, 8275, 8276, 8274, 7938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=10240 expected_recv=15869 expected_overflow=0 exact_recv_per_rank=[7952, 8273, 8275, 8273, 8275, 8276, 8274, 7938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=10240 expected_recv=15869 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7952, 8273, 8275, 8273, 8275, 8276, 8274, 7938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7952, 8273, 8275, 8273, 8275, 8276, 8274, 7938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7952, 8273, 8275, 8273, 8275, 8276, 8274, 7938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8273, 8274, 7954, 7939, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8273, 8274, 7954, 7939, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=10240 expected_recv=15906 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8273, 8274, 7954, 7939, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8273, 8274, 7954, 7939, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8273, 8274, 7954, 7939, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=10240 expected_recv=15906 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8273, 8274, 7954, 7939, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8273, 8274, 7954, 7939, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=10240 expected_recv=15878 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8273, 8274, 7954, 7939, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=10240 expected_recv=15878 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=10240 expected_recv=16228 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=10240 expected_recv=16224 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=10240 expected_recv=16228 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=10240 expected_recv=16224 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=10240 expected_recv=15878 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=10240 expected_recv=15878 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[1039, 6, 4427, 9090, 7937, 8789, 32722, 1526] avg=8192.0000 max=32722 max_rank=6 min=6 min_rank=1 imbalance_ratio=3.994385 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[1039, 6, 4427, 9090, 7937, 8789, 32722, 1526] avg=8192.0000 max=32722 max_rank=6 min=6 min_rank=1 imbalance_ratio=3.994385 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[1039, 6, 4427, 9090, 7937, 8789, 32722, 1526] avg=8192.0000 max=32722 max_rank=6 min=6 min_rank=1 imbalance_ratio=3.994385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[8930, 8930, 4587, 8930, 7937, 8789, 8931, 8502] avg=8192.0000 max=8931 max_rank=6 min=4587 min_rank=2 imbalance_ratio=1.090210 max_load_reduction=23791 max_load_reduction_pct=0.727064 moved_to_replicas=48355 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[1039, 6, 4427, 9090, 7937, 8789, 32722, 1526] avg=8192.0000 max=32722 max_rank=6 min=6 min_rank=1 imbalance_ratio=3.994385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[1039, 6, 4427, 9090, 7937, 8789, 32722, 1526] avg=8192.0000 max=32722 max_rank=6 min=6 min_rank=1 imbalance_ratio=3.994385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[8930, 8930, 4587, 8930, 7937, 8789, 8931, 8502] avg=8192.0000 max=8931 max_rank=6 min=4587 min_rank=2 imbalance_ratio=1.090210 max_load_reduction=23791 max_load_reduction_pct=0.727064 moved_to_replicas=48355 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[1039, 6, 4427, 9090, 7937, 8789, 32722, 1526] avg=8192.0000 max=32722 max_rank=6 min=6 min_rank=1 imbalance_ratio=3.994385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[1039, 6, 4427, 9090, 7937, 8789, 32722, 1526] avg=8192.0000 max=32722 max_rank=6 min=6 min_rank=1 imbalance_ratio=3.994385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[1039, 6, 4427, 9090, 7937, 8789, 32722, 1526] avg=8192.0000 max=32722 max_rank=6 min=6 min_rank=1 imbalance_ratio=3.994385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[8930, 8930, 4587, 8930, 7937, 8789, 8931, 8502] avg=8192.0000 max=8931 max_rank=6 min=4587 min_rank=2 imbalance_ratio=1.090210 max_load_reduction=23791 max_load_reduction_pct=0.727064 moved_to_replicas=48355 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[8930, 8930, 4587, 8930, 7937, 8789, 8931, 8502] avg=8192.0000 max=8931 max_rank=6 min=4587 min_rank=2 imbalance_ratio=1.090210 max_load_reduction=23791 max_load_reduction_pct=0.727064 moved_to_replicas=48355 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[8930, 8930, 4587, 8930, 7937, 8789, 8931, 8502] avg=8192.0000 max=8931 max_rank=6 min=4587 min_rank=2 imbalance_ratio=1.090210 max_load_reduction=23791 max_load_reduction_pct=0.727064 moved_to_replicas=48355 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[2, 6, 90, 0, 1385, 32761, 31292, 0] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=3 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[8930, 8930, 4587, 8930, 7937, 8789, 8931, 8502] avg=8192.0000 max=8931 max_rank=6 min=4587 min_rank=2 imbalance_ratio=1.090210 max_load_reduction=23791 max_load_reduction_pct=0.727064 moved_to_replicas=48355 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[2, 6, 90, 0, 1385, 32761, 31292, 0] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=3 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[8930, 8930, 4587, 8930, 7937, 8789, 8931, 8502] avg=8192.0000 max=8931 max_rank=6 min=4587 min_rank=2 imbalance_ratio=1.090210 max_load_reduction=23791 max_load_reduction_pct=0.727064 moved_to_replicas=48355 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[8930, 8930, 4587, 8930, 7937, 8789, 8931, 8502] avg=8192.0000 max=8931 max_rank=6 min=4587 min_rank=2 imbalance_ratio=1.090210 max_load_reduction=23791 max_load_reduction_pct=0.727064 moved_to_replicas=48355 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[2, 6, 90, 0, 1385, 32761, 31292, 0] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=3 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[7941, 8273, 8275, 8275, 7951, 8274, 8274, 8273] avg=8192.0000 max=8275 max_rank=2 min=7941 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=94954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[2, 6, 90, 0, 1385, 32761, 31292, 0] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=3 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[2, 6, 90, 0, 1385, 32761, 31292, 0] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=3 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[7941, 8273, 8275, 8275, 7951, 8274, 8274, 8273] avg=8192.0000 max=8275 max_rank=2 min=7941 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=94954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[2, 6, 90, 0, 1385, 32761, 31292, 0] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=3 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[7941, 8273, 8275, 8275, 7951, 8274, 8274, 8273] avg=8192.0000 max=8275 max_rank=2 min=7941 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=94954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[2, 6, 90, 0, 1385, 32761, 31292, 0] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=3 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[2, 6, 90, 0, 1385, 32761, 31292, 0] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=3 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[7941, 8273, 8275, 8275, 7951, 8274, 8274, 8273] avg=8192.0000 max=8275 max_rank=2 min=7941 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=94954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[7941, 8273, 8275, 8275, 7951, 8274, 8274, 8273] avg=8192.0000 max=8275 max_rank=2 min=7941 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=94954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[2, 32756, 82, 5, 1, 32688, 0, 2] avg=8192.0000 max=32756 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[7941, 8273, 8275, 8275, 7951, 8274, 8274, 8273] avg=8192.0000 max=8275 max_rank=2 min=7941 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=94954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[2, 32756, 82, 5, 1, 32688, 0, 2] avg=8192.0000 max=32756 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[7941, 8273, 8275, 8275, 7951, 8274, 8274, 8273] avg=8192.0000 max=8275 max_rank=2 min=7941 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=94954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[7941, 8273, 8275, 8275, 7951, 8274, 8274, 8273] avg=8192.0000 max=8275 max_rank=2 min=7941 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=94954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[7937, 8275, 7955, 8275, 8273, 8274, 8274, 8273] avg=8192.0000 max=8275 max_rank=1 min=7937 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97662 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[2, 32756, 82, 5, 1, 32688, 0, 2] avg=8192.0000 max=32756 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[2, 32756, 82, 5, 1, 32688, 0, 2] avg=8192.0000 max=32756 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[2, 32756, 82, 5, 1, 32688, 0, 2] avg=8192.0000 max=32756 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[7937, 8275, 7955, 8275, 8273, 8274, 8274, 8273] avg=8192.0000 max=8275 max_rank=1 min=7937 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97662 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[2, 32756, 82, 5, 1, 32688, 0, 2] avg=8192.0000 max=32756 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[7937, 8275, 7955, 8275, 8273, 8274, 8274, 8273] avg=8192.0000 max=8275 max_rank=1 min=7937 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97662 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[2, 32756, 82, 5, 1, 32688, 0, 2] avg=8192.0000 max=32756 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[7937, 8275, 7955, 8275, 8273, 8274, 8274, 8273] avg=8192.0000 max=8275 max_rank=1 min=7937 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97662 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[2, 32756, 82, 5, 1, 32688, 0, 2] avg=8192.0000 max=32756 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[7937, 8275, 7955, 8275, 8273, 8274, 8274, 8273] avg=8192.0000 max=8275 max_rank=1 min=7937 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97662 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[32763, 32761, 4, 2, 2, 1, 1, 2] avg=8192.0000 max=32763 max_rank=0 min=1 min_rank=5 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[7937, 8275, 7955, 8275, 8273, 8274, 8274, 8273] avg=8192.0000 max=8275 max_rank=1 min=7937 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97662 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[32763, 32761, 4, 2, 2, 1, 1, 2] avg=8192.0000 max=32763 max_rank=0 min=1 min_rank=5 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[7937, 8275, 7955, 8275, 8273, 8274, 8274, 8273] avg=8192.0000 max=8275 max_rank=1 min=7937 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97662 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[7937, 8275, 7955, 8275, 8273, 8274, 8274, 8273] avg=8192.0000 max=8275 max_rank=1 min=7937 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97662 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[8275, 8274, 7948, 7945, 8273, 8275, 8272, 8274] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[32763, 32761, 4, 2, 2, 1, 1, 2] avg=8192.0000 max=32763 max_rank=0 min=1 min_rank=5 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[32763, 32761, 4, 2, 2, 1, 1, 2] avg=8192.0000 max=32763 max_rank=0 min=1 min_rank=5 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[32763, 32761, 4, 2, 2, 1, 1, 2] avg=8192.0000 max=32763 max_rank=0 min=1 min_rank=5 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[8275, 8274, 7948, 7945, 8273, 8275, 8272, 8274] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[32763, 32761, 4, 2, 2, 1, 1, 2] avg=8192.0000 max=32763 max_rank=0 min=1 min_rank=5 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[8275, 8274, 7948, 7945, 8273, 8275, 8272, 8274] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[32763, 32761, 4, 2, 2, 1, 1, 2] avg=8192.0000 max=32763 max_rank=0 min=1 min_rank=5 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[8275, 8274, 7948, 7945, 8273, 8275, 8272, 8274] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[8275, 8274, 7948, 7945, 8273, 8275, 8272, 8274] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[32763, 32761, 4, 2, 2, 1, 1, 2] avg=8192.0000 max=32763 max_rank=0 min=1 min_rank=5 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[32764, 1, 10341, 22422, 4, 4, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[8275, 8274, 7948, 7945, 8273, 8275, 8272, 8274] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[32764, 1, 10341, 22422, 4, 4, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[8275, 8274, 7948, 7945, 8273, 8275, 8272, 8274] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[8275, 8274, 7948, 7945, 8273, 8275, 8272, 8274] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[11211, 11211, 10341, 11212, 4, 4, 11211, 10342] avg=8192.0000 max=11212 max_rank=3 min=4 min_rank=4 imbalance_ratio=1.368652 max_load_reduction=21552 max_load_reduction_pct=0.657795 moved_to_replicas=62515 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[32764, 1, 10341, 22422, 4, 4, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[32764, 1, 10341, 22422, 4, 4, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[32764, 1, 10341, 22422, 4, 4, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[11211, 11211, 10341, 11212, 4, 4, 11211, 10342] avg=8192.0000 max=11212 max_rank=3 min=4 min_rank=4 imbalance_ratio=1.368652 max_load_reduction=21552 max_load_reduction_pct=0.657795 moved_to_replicas=62515 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[32764, 1, 10341, 22422, 4, 4, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[11211, 11211, 10341, 11212, 4, 4, 11211, 10342] avg=8192.0000 max=11212 max_rank=3 min=4 min_rank=4 imbalance_ratio=1.368652 max_load_reduction=21552 max_load_reduction_pct=0.657795 moved_to_replicas=62515 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[32764, 1, 10341, 22422, 4, 4, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[11211, 11211, 10341, 11212, 4, 4, 11211, 10342] avg=8192.0000 max=11212 max_rank=3 min=4 min_rank=4 imbalance_ratio=1.368652 max_load_reduction=21552 max_load_reduction_pct=0.657795 moved_to_replicas=62515 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[11211, 11211, 10341, 11212, 4, 4, 11211, 10342] avg=8192.0000 max=11212 max_rank=3 min=4 min_rank=4 imbalance_ratio=1.368652 max_load_reduction=21552 max_load_reduction_pct=0.657795 moved_to_replicas=62515 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[32764, 1, 10341, 22422, 4, 4, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[32764, 4, 32763, 0, 0, 1, 3, 1] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[11211, 11211, 10341, 11212, 4, 4, 11211, 10342] avg=8192.0000 max=11212 max_rank=3 min=4 min_rank=4 imbalance_ratio=1.368652 max_load_reduction=21552 max_load_reduction_pct=0.657795 moved_to_replicas=62515 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[32764, 4, 32763, 0, 0, 1, 3, 1] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[11211, 11211, 10341, 11212, 4, 4, 11211, 10342] avg=8192.0000 max=11212 max_rank=3 min=4 min_rank=4 imbalance_ratio=1.368652 max_load_reduction=21552 max_load_reduction_pct=0.657795 moved_to_replicas=62515 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[11211, 11211, 10341, 11212, 4, 4, 11211, 10342] avg=8192.0000 max=11212 max_rank=3 min=4 min_rank=4 imbalance_ratio=1.368652 max_load_reduction=21552 max_load_reduction_pct=0.657795 moved_to_replicas=62515 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[8275, 7948, 8276, 8275, 8274, 7941, 8275, 8272] avg=8192.0000 max=8276 max_rank=2 min=7941 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[32764, 4, 32763, 0, 0, 1, 3, 1] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[32764, 4, 32763, 0, 0, 1, 3, 1] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[32764, 4, 32763, 0, 0, 1, 3, 1] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[8275, 7948, 8276, 8275, 8274, 7941, 8275, 8272] avg=8192.0000 max=8276 max_rank=2 min=7941 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[32764, 4, 32763, 0, 0, 1, 3, 1] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[8275, 7948, 8276, 8275, 8274, 7941, 8275, 8272] avg=8192.0000 max=8276 max_rank=2 min=7941 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[8275, 7948, 8276, 8275, 8274, 7941, 8275, 8272] avg=8192.0000 max=8276 max_rank=2 min=7941 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[32764, 4, 32763, 0, 0, 1, 3, 1] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[8275, 7948, 8276, 8275, 8274, 7941, 8275, 8272] avg=8192.0000 max=8276 max_rank=2 min=7941 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[8275, 7948, 8276, 8275, 8274, 7941, 8275, 8272] avg=8192.0000 max=8276 max_rank=2 min=7941 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[32761, 0, 1, 4, 2, 32764, 0, 4] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[32764, 4, 32763, 0, 0, 1, 3, 1] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[32761, 0, 1, 4, 2, 32764, 0, 4] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[8275, 7948, 8276, 8275, 8274, 7941, 8275, 8272] avg=8192.0000 max=8276 max_rank=2 min=7941 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[32761, 0, 1, 4, 2, 32764, 0, 4] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[32761, 0, 1, 4, 2, 32764, 0, 4] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[8276, 8275, 7941, 8276, 8272, 8275, 8274, 7947] avg=8192.0000 max=8276 max_rank=0 min=7941 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[32761, 0, 1, 4, 2, 32764, 0, 4] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[8275, 7948, 8276, 8275, 8274, 7941, 8275, 8272] avg=8192.0000 max=8276 max_rank=2 min=7941 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[8276, 8275, 7941, 8276, 8272, 8275, 8274, 7947] avg=8192.0000 max=8276 max_rank=0 min=7941 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[32761, 0, 1, 4, 2, 32764, 0, 4] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[8276, 8275, 7941, 8276, 8272, 8275, 8274, 7947] avg=8192.0000 max=8276 max_rank=0 min=7941 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[8276, 8275, 7941, 8276, 8272, 8275, 8274, 7947] avg=8192.0000 max=8276 max_rank=0 min=7941 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[8276, 8275, 7941, 8276, 8272, 8275, 8274, 7947] avg=8192.0000 max=8276 max_rank=0 min=7941 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[32761, 0, 1, 4, 2, 32764, 0, 4] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[8276, 8275, 7941, 8276, 8272, 8275, 8274, 7947] avg=8192.0000 max=8276 max_rank=0 min=7941 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[0, 0, 5, 32759, 5, 62, 32702, 3] avg=8192.0000 max=32759 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[32761, 0, 1, 4, 2, 32764, 0, 4] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[0, 0, 5, 32759, 5, 62, 32702, 3] avg=8192.0000 max=32759 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[8276, 8275, 7941, 8276, 8272, 8275, 8274, 7947] avg=8192.0000 max=8276 max_rank=0 min=7941 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[0, 0, 5, 32759, 5, 62, 32702, 3] avg=8192.0000 max=32759 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[0, 0, 5, 32759, 5, 62, 32702, 3] avg=8192.0000 max=32759 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[8276, 8276, 8275, 8272, 8274, 7952, 8273, 7938] avg=8192.0000 max=8276 max_rank=0 min=7938 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97866 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[0, 0, 5, 32759, 5, 62, 32702, 3] avg=8192.0000 max=32759 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[8276, 8275, 7941, 8276, 8272, 8275, 8274, 7947] avg=8192.0000 max=8276 max_rank=0 min=7941 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[8276, 8276, 8275, 8272, 8274, 7952, 8273, 7938] avg=8192.0000 max=8276 max_rank=0 min=7938 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97866 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[0, 0, 5, 32759, 5, 62, 32702, 3] avg=8192.0000 max=32759 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[8276, 8276, 8275, 8272, 8274, 7952, 8273, 7938] avg=8192.0000 max=8276 max_rank=0 min=7938 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97866 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[8276, 8276, 8275, 8272, 8274, 7952, 8273, 7938] avg=8192.0000 max=8276 max_rank=0 min=7938 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97866 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[8276, 8276, 8275, 8272, 8274, 7952, 8273, 7938] avg=8192.0000 max=8276 max_rank=0 min=7938 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97866 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[0, 0, 5, 32759, 5, 62, 32702, 3] avg=8192.0000 max=32759 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[8276, 8276, 8275, 8272, 8274, 7952, 8273, 7938] avg=8192.0000 max=8276 max_rank=0 min=7938 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97866 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[0, 32760, 4, 1, 5, 32762, 1, 3] avg=8192.0000 max=32762 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[0, 0, 5, 32759, 5, 62, 32702, 3] avg=8192.0000 max=32759 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[0, 32760, 4, 1, 5, 32762, 1, 3] avg=8192.0000 max=32762 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[8276, 8276, 8275, 8272, 8274, 7952, 8273, 7938] avg=8192.0000 max=8276 max_rank=0 min=7938 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97866 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[0, 32760, 4, 1, 5, 32762, 1, 3] avg=8192.0000 max=32762 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[0, 32760, 4, 1, 5, 32762, 1, 3] avg=8192.0000 max=32762 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[0, 32760, 4, 1, 5, 32762, 1, 3] avg=8192.0000 max=32762 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[8276, 8276, 8275, 8272, 8274, 7952, 8273, 7938] avg=8192.0000 max=8276 max_rank=0 min=7938 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97866 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[0, 32760, 4, 1, 5, 32762, 1, 3] avg=8192.0000 max=32762 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[0, 32760, 4, 1, 5, 32762, 1, 3] avg=8192.0000 max=32762 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[0, 32763, 5, 5, 1, 0, 32762, 0] avg=8192.0000 max=32763 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[0, 32760, 4, 1, 5, 32762, 1, 3] avg=8192.0000 max=32762 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[0, 32763, 5, 5, 1, 0, 32762, 0] avg=8192.0000 max=32763 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[0, 32763, 5, 5, 1, 0, 32762, 0] avg=8192.0000 max=32763 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[0, 32763, 5, 5, 1, 0, 32762, 0] avg=8192.0000 max=32763 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[8276, 8276, 8272, 7952, 8275, 8272, 8274, 7939] avg=8192.0000 max=8276 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[0, 32763, 5, 5, 1, 0, 32762, 0] avg=8192.0000 max=32763 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[0, 32763, 5, 5, 1, 0, 32762, 0] avg=8192.0000 max=32763 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[8276, 8276, 8272, 7952, 8275, 8272, 8274, 7939] avg=8192.0000 max=8276 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[8276, 8276, 8272, 7952, 8275, 8272, 8274, 7939] avg=8192.0000 max=8276 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[8276, 8276, 8272, 7952, 8275, 8272, 8274, 7939] avg=8192.0000 max=8276 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[8276, 8276, 8272, 7952, 8275, 8272, 8274, 7939] avg=8192.0000 max=8276 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[8276, 8276, 8272, 7952, 8275, 8272, 8274, 7939] avg=8192.0000 max=8276 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[0, 32763, 5, 5, 1, 0, 32762, 0] avg=8192.0000 max=32763 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[6, 0, 0, 0, 4, 5, 32759, 32762] avg=8192.0000 max=32762 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[0, 32763, 5, 5, 1, 0, 32762, 0] avg=8192.0000 max=32763 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[6, 0, 0, 0, 4, 5, 32759, 32762] avg=8192.0000 max=32762 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[6, 0, 0, 0, 4, 5, 32759, 32762] avg=8192.0000 max=32762 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[8276, 8276, 8272, 7952, 8275, 8272, 8274, 7939] avg=8192.0000 max=8276 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[6, 0, 0, 0, 4, 5, 32759, 32762] avg=8192.0000 max=32762 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[6, 0, 0, 0, 4, 5, 32759, 32762] avg=8192.0000 max=32762 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[8276, 8276, 8272, 7952, 8275, 8272, 8274, 7939] avg=8192.0000 max=8276 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[6, 0, 0, 0, 4, 5, 32759, 32762] avg=8192.0000 max=32762 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[2, 32030, 6, 32755, 2, 0, 6, 735] avg=8192.0000 max=32755 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[6, 0, 0, 0, 4, 5, 32759, 32762] avg=8192.0000 max=32762 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[6, 0, 0, 0, 4, 5, 32759, 32762] avg=8192.0000 max=32762 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[2, 32030, 6, 32755, 2, 0, 6, 735] avg=8192.0000 max=32755 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[2, 32030, 6, 32755, 2, 0, 6, 735] avg=8192.0000 max=32755 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[8275, 8273, 8274, 8274, 7937, 8273, 8274, 7956] avg=8192.0000 max=8275 max_rank=0 min=7937 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=96269 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[2, 32030, 6, 32755, 2, 0, 6, 735] avg=8192.0000 max=32755 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[2, 32030, 6, 32755, 2, 0, 6, 735] avg=8192.0000 max=32755 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[2, 32030, 6, 32755, 2, 0, 6, 735] avg=8192.0000 max=32755 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[8275, 8273, 8274, 8274, 7937, 8273, 8274, 7956] avg=8192.0000 max=8275 max_rank=0 min=7937 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=96269 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[8275, 8273, 8274, 8274, 7937, 8273, 8274, 7956] avg=8192.0000 max=8275 max_rank=0 min=7937 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=96269 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[8275, 8273, 8274, 8274, 7937, 8273, 8274, 7956] avg=8192.0000 max=8275 max_rank=0 min=7937 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=96269 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[8275, 8273, 8274, 8274, 7937, 8273, 8274, 7956] avg=8192.0000 max=8275 max_rank=0 min=7937 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=96269 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[0, 2, 0, 6, 32762, 3, 5, 32758] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[8275, 8273, 8274, 8274, 7937, 8273, 8274, 7956] avg=8192.0000 max=8275 max_rank=0 min=7937 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=96269 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[2, 32030, 6, 32755, 2, 0, 6, 735] avg=8192.0000 max=32755 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[2, 32030, 6, 32755, 2, 0, 6, 735] avg=8192.0000 max=32755 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[0, 2, 0, 6, 32762, 3, 5, 32758] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[8275, 7941, 8274, 7950, 8274, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[0, 2, 0, 6, 32762, 3, 5, 32758] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[8275, 8273, 8274, 8274, 7937, 8273, 8274, 7956] avg=8192.0000 max=8275 max_rank=0 min=7937 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=96269 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[0, 2, 0, 6, 32762, 3, 5, 32758] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[0, 2, 0, 6, 32762, 3, 5, 32758] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[8275, 8273, 8274, 8274, 7937, 8273, 8274, 7956] avg=8192.0000 max=8275 max_rank=0 min=7937 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=96269 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[0, 2, 0, 6, 32762, 3, 5, 32758] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[8275, 7941, 8274, 7950, 8274, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[8275, 7941, 8274, 7950, 8274, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[8275, 7941, 8274, 7950, 8274, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[8275, 7941, 8274, 7950, 8274, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[59, 1, 32704, 8, 0, 32759, 3, 2] avg=8192.0000 max=32759 max_rank=5 min=0 min_rank=4 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[8275, 7941, 8274, 7950, 8274, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[0, 2, 0, 6, 32762, 3, 5, 32758] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[0, 2, 0, 6, 32762, 3, 5, 32758] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[59, 1, 32704, 8, 0, 32759, 3, 2] avg=8192.0000 max=32759 max_rank=5 min=0 min_rank=4 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[7952, 8273, 8275, 8273, 8275, 8276, 8274, 7938] avg=8192.0000 max=8276 max_rank=5 min=7938 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97806 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[59, 1, 32704, 8, 0, 32759, 3, 2] avg=8192.0000 max=32759 max_rank=5 min=0 min_rank=4 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[8275, 7941, 8274, 7950, 8274, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[59, 1, 32704, 8, 0, 32759, 3, 2] avg=8192.0000 max=32759 max_rank=5 min=0 min_rank=4 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[59, 1, 32704, 8, 0, 32759, 3, 2] avg=8192.0000 max=32759 max_rank=5 min=0 min_rank=4 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[8275, 7941, 8274, 7950, 8274, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[7952, 8273, 8275, 8273, 8275, 8276, 8274, 7938] avg=8192.0000 max=8276 max_rank=5 min=7938 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97806 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[59, 1, 32704, 8, 0, 32759, 3, 2] avg=8192.0000 max=32759 max_rank=5 min=0 min_rank=4 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[7952, 8273, 8275, 8273, 8275, 8276, 8274, 7938] avg=8192.0000 max=8276 max_rank=5 min=7938 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97806 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[6, 0, 0, 1, 32762, 6, 0, 32761] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[7952, 8273, 8275, 8273, 8275, 8276, 8274, 7938] avg=8192.0000 max=8276 max_rank=5 min=7938 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97806 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[7952, 8273, 8275, 8273, 8275, 8276, 8274, 7938] avg=8192.0000 max=8276 max_rank=5 min=7938 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97806 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[7952, 8273, 8275, 8273, 8275, 8276, 8274, 7938] avg=8192.0000 max=8276 max_rank=5 min=7938 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97806 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[59, 1, 32704, 8, 0, 32759, 3, 2] avg=8192.0000 max=32759 max_rank=5 min=0 min_rank=4 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[6, 0, 0, 1, 32762, 6, 0, 32761] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[59, 1, 32704, 8, 0, 32759, 3, 2] avg=8192.0000 max=32759 max_rank=5 min=0 min_rank=4 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[8274, 8275, 8274, 8273, 8274, 7954, 7939, 8273] avg=8192.0000 max=8275 max_rank=1 min=7939 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[6, 0, 0, 1, 32762, 6, 0, 32761] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[7952, 8273, 8275, 8273, 8275, 8276, 8274, 7938] avg=8192.0000 max=8276 max_rank=5 min=7938 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97806 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[8274, 8275, 8274, 8273, 8274, 7954, 7939, 8273] avg=8192.0000 max=8275 max_rank=1 min=7939 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[6, 0, 0, 1, 32762, 6, 0, 32761] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[7952, 8273, 8275, 8273, 8275, 8276, 8274, 7938] avg=8192.0000 max=8276 max_rank=5 min=7938 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97806 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[6, 0, 0, 1, 32762, 6, 0, 32761] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[6, 0, 0, 1, 32762, 6, 0, 32761] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[32761, 0, 32754, 0, 0, 7, 5, 9] avg=8192.0000 max=32761 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[8274, 8275, 8274, 8273, 8274, 7954, 7939, 8273] avg=8192.0000 max=8275 max_rank=1 min=7939 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[8274, 8275, 8274, 8273, 8274, 7954, 7939, 8273] avg=8192.0000 max=8275 max_rank=1 min=7939 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[8274, 8275, 8274, 8273, 8274, 7954, 7939, 8273] avg=8192.0000 max=8275 max_rank=1 min=7939 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[8274, 8275, 8274, 8273, 8274, 7954, 7939, 8273] avg=8192.0000 max=8275 max_rank=1 min=7939 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[6, 0, 0, 1, 32762, 6, 0, 32761] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[32761, 0, 32754, 0, 0, 7, 5, 9] avg=8192.0000 max=32761 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[6, 0, 0, 1, 32762, 6, 0, 32761] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24485 max_load_reduction_pct=0.747383 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[32761, 0, 32754, 0, 0, 7, 5, 9] avg=8192.0000 max=32761 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24485 max_load_reduction_pct=0.747383 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[32761, 0, 32754, 0, 0, 7, 5, 9] avg=8192.0000 max=32761 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[8274, 8275, 8274, 8273, 8274, 7954, 7939, 8273] avg=8192.0000 max=8275 max_rank=1 min=7939 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[8274, 8275, 8274, 8273, 8274, 7954, 7939, 8273] avg=8192.0000 max=8275 max_rank=1 min=7939 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[32761, 0, 32754, 0, 0, 7, 5, 9] avg=8192.0000 max=32761 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[32761, 0, 32754, 0, 0, 7, 5, 9] avg=8192.0000 max=32761 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24485 max_load_reduction_pct=0.747383 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24485 max_load_reduction_pct=0.747383 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24485 max_load_reduction_pct=0.747383 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24485 max_load_reduction_pct=0.747383 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[32761, 0, 32754, 0, 0, 7, 5, 9] avg=8192.0000 max=32761 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[32761, 0, 32754, 0, 0, 7, 5, 9] avg=8192.0000 max=32761 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24485 max_load_reduction_pct=0.747383 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24485 max_load_reduction_pct=0.747383 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=32 total_expected_recv=270368 total_dropped_tokens=2657 drop_rate_pct=0.982735 records_with_drop_pct=6.250000 peak_rank_overflow=1685 total_slack_slots=56192 peak_demand=11925 required_capacity=1.455688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=32 total_expected_recv=265055 total_dropped_tokens=102 drop_rate_pct=0.038483 records_with_drop_pct=3.125000 peak_rank_overflow=102 total_slack_slots=58624 peak_demand=10342 required_capacity=1.262451 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=32 total_expected_recv=259113 total_dropped_tokens=971 drop_rate_pct=0.374740 records_with_drop_pct=3.125000 peak_rank_overflow=971 total_slack_slots=66560 peak_demand=11211 required_capacity=1.368530 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=32 total_expected_recv=270632 total_dropped_tokens=2654 drop_rate_pct=0.980667 records_with_drop_pct=6.250000 peak_rank_overflow=1683 total_slack_slots=56576 peak_demand=11923 required_capacity=1.455444 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=32 total_expected_recv=254022 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=69632 peak_demand=8915 required_capacity=1.088257 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=32 total_expected_recv=258288 total_dropped_tokens=1686 drop_rate_pct=0.652760 records_with_drop_pct=3.125000 peak_rank_overflow=1686 total_slack_slots=67584 peak_demand=11926 required_capacity=1.455811 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=32 total_expected_recv=259745 total_dropped_tokens=971 drop_rate_pct=0.373828 records_with_drop_pct=3.125000 peak_rank_overflow=971 total_slack_slots=65408 peak_demand=11211 required_capacity=1.368530 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=32 total_expected_recv=259929 total_dropped_tokens=1786 drop_rate_pct=0.687111 records_with_drop_pct=6.250000 peak_rank_overflow=1685 total_slack_slots=65792 peak_demand=11925 required_capacity=1.455688 + [2026-08-30 05:08:01] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 900.0 | throughput per GPU (TFLOP/s/GPU): 370.4 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.007393E+01 | loss scale: 1.0 | grad norm: 40.104 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 10240 | min safe capacity: 1.2500 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=10240 expected_recv=12297 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 8930, 5234, 7494, 8158, 8930, 8930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=10240 expected_recv=17859 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 8930, 5234, 7494, 8158, 8930, 8930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=10240 expected_recv=17857 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 8930, 5234, 7494, 8158, 8930, 8930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=10240 expected_recv=17858 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 8930, 5234, 7494, 8158, 8930, 8930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=10240 expected_recv=13849 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 8930, 5234, 7494, 8158, 8930, 8930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=10240 expected_recv=17857 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 8930, 5234, 7494, 8158, 8930, 8930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=10240 expected_recv=17858 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 8930, 5234, 7494, 8158, 8930, 8930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=10240 expected_recv=12297 actual_valid_recv=12544 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=10240 expected_recv=17859 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=10240 expected_recv=17857 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=10240 expected_recv=17858 actual_valid_recv=18176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=10240 expected_recv=15637 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 8930, 5234, 7494, 8158, 8930, 8930] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=10240 expected_recv=13849 actual_valid_recv=14080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=12544 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17920 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=10240 expected_recv=17857 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=18176 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17920 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=10240 expected_recv=15637 actual_valid_recv=16000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=10240 expected_recv=17858 actual_valid_recv=18176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=14080 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17920 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=16000 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=18176 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=10240 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8274, 7938, 8274, 8274, 8005, 8273, 8275, 8223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=10240 expected_recv=16220 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 7938, 8274, 8274, 8005, 8273, 8275, 8223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=10240 expected_recv=16212 expected_overflow=0 exact_recv_per_rank=[8274, 7938, 8274, 8274, 8005, 8273, 8275, 8223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 7938, 8274, 8274, 8005, 8273, 8275, 8223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8274, 7938, 8274, 8274, 8005, 8273, 8275, 8223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=10240 expected_recv=16212 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 7938, 8274, 8274, 8005, 8273, 8275, 8223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=10240 expected_recv=15953 expected_overflow=0 exact_recv_per_rank=[8274, 7938, 8274, 8274, 8005, 8273, 8275, 8223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=10240 expected_recv=16496 expected_overflow=0 exact_recv_per_rank=[8274, 7938, 8274, 8274, 8005, 8273, 8275, 8223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=10240 expected_recv=15953 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=10240 expected_recv=16496 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=10240 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 7945, 8273, 8274, 7948, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=10240 expected_recv=16217 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 7945, 8273, 8274, 7948, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 7945, 8273, 8274, 7948, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 7945, 8273, 8274, 7948, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 7945, 8273, 8274, 7948, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 7945, 8273, 8274, 7948, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=10240 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 7945, 8273, 8274, 7948, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=10240 expected_recv=16223 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 7945, 8273, 8274, 7948, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7946, 8275, 8273, 8276, 8273, 8275, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=10240 expected_recv=16220 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=10240 expected_recv=16223 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7946, 8275, 8273, 8276, 8273, 8275, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[7946, 8275, 8273, 8276, 8273, 8275, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7946, 8275, 8273, 8276, 8273, 8275, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=10240 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[7946, 8275, 8273, 8276, 8273, 8275, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7946, 8275, 8273, 8276, 8273, 8275, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=10240 expected_recv=16221 expected_overflow=0 exact_recv_per_rank=[7946, 8275, 8273, 8276, 8273, 8275, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=10240 expected_recv=16217 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 8275, 7947, 7943, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=10240 expected_recv=16221 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[7946, 8275, 8273, 8276, 8273, 8275, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=10240 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 8275, 7947, 7943, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 8275, 7947, 7943, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=10240 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 8275, 7947, 7943, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=10240 expected_recv=16216 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=10240 expected_recv=16215 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 8275, 7947, 7943, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 8275, 7947, 7943, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=10240 expected_recv=16545 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=10240 expected_recv=15900 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 8275, 7947, 7943, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=10240 expected_recv=16215 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 8275, 7947, 7943, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[7948, 8274, 8275, 7946, 8272, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=10240 expected_recv=15900 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7948, 8274, 8275, 7946, 8272, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=10240 expected_recv=15897 expected_overflow=0 exact_recv_per_rank=[7948, 8274, 8275, 7946, 8272, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=10240 expected_recv=15897 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7948, 8274, 8275, 7946, 8272, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7948, 8274, 8275, 7946, 8272, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7948, 8274, 8275, 7946, 8272, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=10240 expected_recv=16213 expected_overflow=0 exact_recv_per_rank=[7948, 8274, 8275, 7946, 8272, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7948, 8274, 8275, 7946, 8272, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=10240 expected_recv=16213 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=10240 expected_recv=16227 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=10240 expected_recv=16215 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=10240 expected_recv=16213 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=10240 expected_recv=16227 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=10240 expected_recv=16215 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=10240 expected_recv=16212 expected_overflow=0 exact_recv_per_rank=[7949, 8276, 8274, 8272, 8274, 8275, 8274, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=10240 expected_recv=16223 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=10240 expected_recv=16213 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=10240 expected_recv=16212 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=10240 expected_recv=16223 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7949, 8276, 8274, 8272, 8274, 8275, 8274, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=10240 expected_recv=15900 expected_overflow=0 exact_recv_per_rank=[7949, 8276, 8274, 8272, 8274, 8275, 8274, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=10240 expected_recv=15900 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7949, 8276, 8274, 8272, 8274, 8275, 8274, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7949, 8276, 8274, 8272, 8274, 8275, 8274, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7949, 8276, 8274, 8272, 8274, 8275, 8274, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7949, 8276, 8274, 8272, 8274, 8275, 8274, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=10240 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=10240 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[7949, 8276, 8274, 8272, 8274, 8275, 8274, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=10240 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=10240 expected_recv=16217 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=10240 expected_recv=15883 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=10240 expected_recv=15883 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=10240 expected_recv=15898 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=10240 expected_recv=15901 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8275, 7947, 7944, 8274, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=10240 expected_recv=15898 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=10240 expected_recv=15901 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8275, 7947, 7944, 8274, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8275, 7947, 7944, 8274, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8275, 7947, 7944, 8274, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8275, 7947, 7944, 8274, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8275, 7947, 7944, 8274, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 8273, 7951, 7943, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=10240 expected_recv=15883 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8275, 7947, 7944, 8274, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8275, 7947, 7944, 8274, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 8273, 7951, 7943, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=10240 expected_recv=15883 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 8273, 7951, 7943, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=10240 expected_recv=15882 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 8273, 7951, 7943, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=10240 expected_recv=15882 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 8273, 7951, 7943, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8273, 7940, 8274, 7953, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 8273, 7951, 7943, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=10240 expected_recv=15903 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 8273, 7951, 7943, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 8273, 7951, 7943, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8273, 7940, 8274, 7953, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=10240 expected_recv=15903 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8273, 7940, 8274, 7953, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8273, 7940, 8274, 7953, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8274, 8275, 8272, 7949, 8274, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8273, 7940, 8274, 7953, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=10240 expected_recv=15910 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8273, 7940, 8274, 7953, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=10240 expected_recv=15875 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8273, 7940, 8274, 7953, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8273, 7940, 8274, 7953, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=10240 expected_recv=16214 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8274, 8275, 8272, 7949, 8274, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=10240 expected_recv=15910 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=10240 expected_recv=15875 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=10240 expected_recv=16214 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8274, 8275, 8272, 7949, 8274, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=10240 expected_recv=15901 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8274, 8275, 8272, 7949, 8274, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=10240 expected_recv=15901 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=10240 expected_recv=15902 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8274, 8275, 8272, 7949, 8274, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=10240 expected_recv=15902 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8274, 8275, 8272, 7949, 8274, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=10240 expected_recv=16215 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8274, 8275, 8272, 7949, 8274, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8274, 8275, 8272, 7949, 8274, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=10240 expected_recv=16215 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=10240 expected_recv=16213 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=10240 expected_recv=16213 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=10240 expected_recv=16213 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=10240 expected_recv=16213 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=10240 expected_recv=15903 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=10240 expected_recv=15903 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8276, 8274, 7940, 8275, 7948, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=10240 expected_recv=15880 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8276, 8274, 7940, 8275, 7948, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=10240 expected_recv=15880 actual_valid_recv=16000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8276, 8274, 7940, 8275, 7948, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16000 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8276, 8274, 7940, 8275, 7948, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[13984, 17, 10307, 3857, 2440, 2201, 32727, 3] avg=8192.0000 max=32727 max_rank=6 min=3 min_rank=7 imbalance_ratio=3.994995 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8276, 8274, 7940, 8275, 7948, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8276, 8274, 7940, 8275, 7948, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=10240 expected_recv=15903 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8276, 8274, 7940, 8275, 7948, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=10240 expected_recv=15876 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8276, 8274, 7940, 8275, 7948, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[8930, 8930, 8930, 5234, 7494, 8158, 8930, 8930] avg=8192.0000 max=8930 max_rank=0 min=5234 min_rank=3 imbalance_ratio=1.090088 max_load_reduction=23797 max_load_reduction_pct=0.727137 moved_to_replicas=60315 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=10240 expected_recv=15903 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=10240 expected_recv=15876 actual_valid_recv=16000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[13984, 17, 10307, 3857, 2440, 2201, 32727, 3] avg=8192.0000 max=32727 max_rank=6 min=3 min_rank=7 imbalance_ratio=3.994995 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[13984, 17, 10307, 3857, 2440, 2201, 32727, 3] avg=8192.0000 max=32727 max_rank=6 min=3 min_rank=7 imbalance_ratio=3.994995 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16000 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[13984, 17, 10307, 3857, 2440, 2201, 32727, 3] avg=8192.0000 max=32727 max_rank=6 min=3 min_rank=7 imbalance_ratio=3.994995 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[8930, 8930, 8930, 5234, 7494, 8158, 8930, 8930] avg=8192.0000 max=8930 max_rank=0 min=5234 min_rank=3 imbalance_ratio=1.090088 max_load_reduction=23797 max_load_reduction_pct=0.727137 moved_to_replicas=60315 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[2, 2, 24762, 4, 8005, 32755, 1, 5] avg=8192.0000 max=32755 max_rank=5 min=1 min_rank=6 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[8930, 8930, 8930, 5234, 7494, 8158, 8930, 8930] avg=8192.0000 max=8930 max_rank=0 min=5234 min_rank=3 imbalance_ratio=1.090088 max_load_reduction=23797 max_load_reduction_pct=0.727137 moved_to_replicas=60315 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[13984, 17, 10307, 3857, 2440, 2201, 32727, 3] avg=8192.0000 max=32727 max_rank=6 min=3 min_rank=7 imbalance_ratio=3.994995 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[8930, 8930, 8930, 5234, 7494, 8158, 8930, 8930] avg=8192.0000 max=8930 max_rank=0 min=5234 min_rank=3 imbalance_ratio=1.090088 max_load_reduction=23797 max_load_reduction_pct=0.727137 moved_to_replicas=60315 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[13984, 17, 10307, 3857, 2440, 2201, 32727, 3] avg=8192.0000 max=32727 max_rank=6 min=3 min_rank=7 imbalance_ratio=3.994995 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[13984, 17, 10307, 3857, 2440, 2201, 32727, 3] avg=8192.0000 max=32727 max_rank=6 min=3 min_rank=7 imbalance_ratio=3.994995 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[13984, 17, 10307, 3857, 2440, 2201, 32727, 3] avg=8192.0000 max=32727 max_rank=6 min=3 min_rank=7 imbalance_ratio=3.994995 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[8274, 7938, 8274, 8274, 8005, 8273, 8275, 8223] avg=8192.0000 max=8275 max_rank=6 min=7938 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=83253 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[8930, 8930, 8930, 5234, 7494, 8158, 8930, 8930] avg=8192.0000 max=8930 max_rank=0 min=5234 min_rank=3 imbalance_ratio=1.090088 max_load_reduction=23797 max_load_reduction_pct=0.727137 moved_to_replicas=60315 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[2, 2, 24762, 4, 8005, 32755, 1, 5] avg=8192.0000 max=32755 max_rank=5 min=1 min_rank=6 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[2, 2, 24762, 4, 8005, 32755, 1, 5] avg=8192.0000 max=32755 max_rank=5 min=1 min_rank=6 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[8930, 8930, 8930, 5234, 7494, 8158, 8930, 8930] avg=8192.0000 max=8930 max_rank=0 min=5234 min_rank=3 imbalance_ratio=1.090088 max_load_reduction=23797 max_load_reduction_pct=0.727137 moved_to_replicas=60315 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[8930, 8930, 8930, 5234, 7494, 8158, 8930, 8930] avg=8192.0000 max=8930 max_rank=0 min=5234 min_rank=3 imbalance_ratio=1.090088 max_load_reduction=23797 max_load_reduction_pct=0.727137 moved_to_replicas=60315 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[8930, 8930, 8930, 5234, 7494, 8158, 8930, 8930] avg=8192.0000 max=8930 max_rank=0 min=5234 min_rank=3 imbalance_ratio=1.090088 max_load_reduction=23797 max_load_reduction_pct=0.727137 moved_to_replicas=60315 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[2, 2, 24762, 4, 8005, 32755, 1, 5] avg=8192.0000 max=32755 max_rank=5 min=1 min_rank=6 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[8274, 7938, 8274, 8274, 8005, 8273, 8275, 8223] avg=8192.0000 max=8275 max_rank=6 min=7938 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=83253 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[2, 1, 32765, 1, 2, 32763, 2, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=7 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[8274, 7938, 8274, 8274, 8005, 8273, 8275, 8223] avg=8192.0000 max=8275 max_rank=6 min=7938 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=83253 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[2, 2, 24762, 4, 8005, 32755, 1, 5] avg=8192.0000 max=32755 max_rank=5 min=1 min_rank=6 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[8274, 7938, 8274, 8274, 8005, 8273, 8275, 8223] avg=8192.0000 max=8275 max_rank=6 min=7938 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=83253 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[2, 2, 24762, 4, 8005, 32755, 1, 5] avg=8192.0000 max=32755 max_rank=5 min=1 min_rank=6 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[2, 2, 24762, 4, 8005, 32755, 1, 5] avg=8192.0000 max=32755 max_rank=5 min=1 min_rank=6 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[2, 2, 24762, 4, 8005, 32755, 1, 5] avg=8192.0000 max=32755 max_rank=5 min=1 min_rank=6 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[8274, 8273, 8275, 7945, 8273, 8274, 7948, 8274] avg=8192.0000 max=8275 max_rank=2 min=7945 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[8274, 7938, 8274, 8274, 8005, 8273, 8275, 8223] avg=8192.0000 max=8275 max_rank=6 min=7938 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=83253 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[2, 1, 32765, 1, 2, 32763, 2, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=7 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[2, 1, 32765, 1, 2, 32763, 2, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=7 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[8274, 7938, 8274, 8274, 8005, 8273, 8275, 8223] avg=8192.0000 max=8275 max_rank=6 min=7938 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=83253 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[2, 1, 32765, 1, 2, 32763, 2, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=7 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[8274, 7938, 8274, 8274, 8005, 8273, 8275, 8223] avg=8192.0000 max=8275 max_rank=6 min=7938 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=83253 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[8274, 7938, 8274, 8274, 8005, 8273, 8275, 8223] avg=8192.0000 max=8275 max_rank=6 min=7938 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=83253 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[8274, 8273, 8275, 7945, 8273, 8274, 7948, 8274] avg=8192.0000 max=8275 max_rank=2 min=7945 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[8274, 8273, 8275, 7945, 8273, 8274, 7948, 8274] avg=8192.0000 max=8275 max_rank=2 min=7945 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[2, 32765, 1, 32760, 2, 1, 2, 3] avg=8192.0000 max=32765 max_rank=1 min=1 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[2, 1, 32765, 1, 2, 32763, 2, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=7 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[8274, 8273, 8275, 7945, 8273, 8274, 7948, 8274] avg=8192.0000 max=8275 max_rank=2 min=7945 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[2, 1, 32765, 1, 2, 32763, 2, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=7 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[2, 1, 32765, 1, 2, 32763, 2, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=7 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[7946, 8275, 8273, 8276, 8273, 8275, 8273, 7945] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[8274, 8273, 8275, 7945, 8273, 8274, 7948, 8274] avg=8192.0000 max=8275 max_rank=2 min=7945 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[2, 1, 32765, 1, 2, 32763, 2, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=7 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[2, 32765, 1, 32760, 2, 1, 2, 3] avg=8192.0000 max=32765 max_rank=1 min=1 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[2, 32765, 1, 32760, 2, 1, 2, 3] avg=8192.0000 max=32765 max_rank=1 min=1 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[8274, 8273, 8275, 7945, 8273, 8274, 7948, 8274] avg=8192.0000 max=8275 max_rank=2 min=7945 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[8274, 8273, 8275, 7945, 8273, 8274, 7948, 8274] avg=8192.0000 max=8275 max_rank=2 min=7945 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[2, 32765, 1, 32760, 2, 1, 2, 3] avg=8192.0000 max=32765 max_rank=1 min=1 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[7946, 8275, 8273, 8276, 8273, 8275, 8273, 7945] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[8274, 8273, 8275, 7945, 8273, 8274, 7948, 8274] avg=8192.0000 max=8275 max_rank=2 min=7945 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[7946, 8275, 8273, 8276, 8273, 8275, 8273, 7945] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[0, 0, 32765, 32765, 3, 0, 2, 1] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[2, 32765, 1, 32760, 2, 1, 2, 3] avg=8192.0000 max=32765 max_rank=1 min=1 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[7946, 8275, 8273, 8276, 8273, 8275, 8273, 7945] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[2, 32765, 1, 32760, 2, 1, 2, 3] avg=8192.0000 max=32765 max_rank=1 min=1 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[2, 32765, 1, 32760, 2, 1, 2, 3] avg=8192.0000 max=32765 max_rank=1 min=1 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[8275, 8275, 8272, 8275, 7947, 7943, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[7946, 8275, 8273, 8276, 8273, 8275, 8273, 7945] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[0, 0, 32765, 32765, 3, 0, 2, 1] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[0, 0, 32765, 32765, 3, 0, 2, 1] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[2, 32765, 1, 32760, 2, 1, 2, 3] avg=8192.0000 max=32765 max_rank=1 min=1 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[7946, 8275, 8273, 8276, 8273, 8275, 8273, 7945] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[7946, 8275, 8273, 8276, 8273, 8275, 8273, 7945] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[0, 0, 32765, 32765, 3, 0, 2, 1] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[8275, 8275, 8272, 8275, 7947, 7943, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[7, 3, 32765, 2, 0, 32756, 3, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[8275, 8275, 8272, 8275, 7947, 7943, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[0, 0, 32765, 32765, 3, 0, 2, 1] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[7946, 8275, 8273, 8276, 8273, 8275, 8273, 7945] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[8275, 8275, 8272, 8275, 7947, 7943, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[7948, 8274, 8275, 7946, 8272, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=2 min=7946 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[0, 0, 32765, 32765, 3, 0, 2, 1] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[0, 0, 32765, 32765, 3, 0, 2, 1] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[8275, 8275, 8272, 8275, 7947, 7943, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[7, 3, 32765, 2, 0, 32756, 3, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[7, 3, 32765, 2, 0, 32756, 3, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[0, 0, 32765, 32765, 3, 0, 2, 1] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[8275, 8275, 8272, 8275, 7947, 7943, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[8275, 8275, 8272, 8275, 7947, 7943, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[7, 3, 32765, 2, 0, 32756, 3, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[32765, 0, 3, 0, 32765, 0, 0, 3] avg=8192.0000 max=32765 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[7948, 8274, 8275, 7946, 8272, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=2 min=7946 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[7948, 8274, 8275, 7946, 8272, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=2 min=7946 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[7, 3, 32765, 2, 0, 32756, 3, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[8275, 8275, 8272, 8275, 7947, 7943, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[7948, 8274, 8275, 7946, 8272, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=2 min=7946 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[7, 3, 32765, 2, 0, 32756, 3, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[7, 3, 32765, 2, 0, 32756, 3, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[7948, 8274, 8275, 7946, 8272, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=2 min=7946 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[32765, 0, 3, 0, 32765, 0, 0, 3] avg=8192.0000 max=32765 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[32765, 0, 3, 0, 32765, 0, 0, 3] avg=8192.0000 max=32765 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[7, 3, 32765, 2, 0, 32756, 3, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[32765, 0, 3, 0, 32765, 0, 0, 3] avg=8192.0000 max=32765 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[7948, 8274, 8275, 7946, 8272, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=2 min=7946 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[44, 0, 3, 0, 3, 32765, 32721, 0] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[7948, 8274, 8275, 7946, 8272, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=2 min=7946 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[7948, 8274, 8275, 7946, 8272, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=2 min=7946 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[32765, 0, 3, 0, 32765, 0, 0, 3] avg=8192.0000 max=32765 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[7949, 8276, 8274, 8272, 8274, 8275, 8274, 7942] avg=8192.0000 max=8276 max_rank=1 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97829 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[32765, 0, 3, 0, 32765, 0, 0, 3] avg=8192.0000 max=32765 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[44, 0, 3, 0, 3, 32765, 32721, 0] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[32765, 0, 3, 0, 32765, 0, 0, 3] avg=8192.0000 max=32765 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[44, 0, 3, 0, 3, 32765, 32721, 0] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[32765, 0, 3, 0, 32765, 0, 0, 3] avg=8192.0000 max=32765 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[44, 0, 3, 0, 3, 32765, 32721, 0] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[3, 32765, 0, 0, 3, 0, 0, 32765] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[7949, 8276, 8274, 8272, 8274, 8275, 8274, 7942] avg=8192.0000 max=8276 max_rank=1 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97829 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[7949, 8276, 8274, 8272, 8274, 8275, 8274, 7942] avg=8192.0000 max=8276 max_rank=1 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97829 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[44, 0, 3, 0, 3, 32765, 32721, 0] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[7949, 8276, 8274, 8272, 8274, 8275, 8274, 7942] avg=8192.0000 max=8276 max_rank=1 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97829 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[44, 0, 3, 0, 3, 32765, 32721, 0] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[3, 32765, 0, 0, 3, 0, 0, 32765] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[7949, 8276, 8274, 8272, 8274, 8275, 8274, 7942] avg=8192.0000 max=8276 max_rank=1 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97829 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[44, 0, 3, 0, 3, 32765, 32721, 0] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[3, 32765, 0, 0, 3, 0, 0, 32765] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[44, 0, 3, 0, 3, 32765, 32721, 0] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[3, 32765, 0, 0, 3, 0, 0, 32765] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[7949, 8276, 8274, 8272, 8274, 8275, 8274, 7942] avg=8192.0000 max=8276 max_rank=1 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97829 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[0, 0, 3, 3, 0, 32765, 32765, 0] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[7949, 8276, 8274, 8272, 8274, 8275, 8274, 7942] avg=8192.0000 max=8276 max_rank=1 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97829 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[7949, 8276, 8274, 8272, 8274, 8275, 8274, 7942] avg=8192.0000 max=8276 max_rank=1 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97829 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[3, 32765, 0, 0, 3, 0, 0, 32765] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[8275, 8272, 8275, 7947, 7944, 8274, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[3, 32765, 0, 0, 3, 0, 0, 32765] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[0, 0, 3, 3, 0, 32765, 32765, 0] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[3, 32765, 0, 0, 3, 0, 0, 32765] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[0, 0, 3, 3, 0, 32765, 32765, 0] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[3, 32765, 0, 0, 3, 0, 0, 32765] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[0, 0, 3, 3, 0, 32765, 32765, 0] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[3, 0, 0, 3, 100, 0, 32765, 32665] avg=8192.0000 max=32765 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[8275, 8272, 8275, 7947, 7944, 8274, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[8275, 8272, 8275, 7947, 7944, 8274, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[0, 0, 3, 3, 0, 32765, 32765, 0] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[8275, 8272, 8275, 7947, 7944, 8274, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[8273, 8275, 8275, 8273, 7951, 7943, 8272, 8274] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97709 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[0, 0, 3, 3, 0, 32765, 32765, 0] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[8275, 8272, 8275, 7947, 7944, 8274, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[3, 0, 0, 3, 100, 0, 32765, 32665] avg=8192.0000 max=32765 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[0, 0, 3, 3, 0, 32765, 32765, 0] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[3, 0, 0, 3, 100, 0, 32765, 32665] avg=8192.0000 max=32765 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[0, 0, 3, 3, 0, 32765, 32765, 0] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[8275, 8272, 8275, 7947, 7944, 8274, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[3, 0, 0, 3, 100, 0, 32765, 32665] avg=8192.0000 max=32765 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[27, 32755, 3, 0, 3, 21, 34, 32693] avg=8192.0000 max=32755 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[8273, 8275, 8275, 8273, 7951, 7943, 8272, 8274] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97709 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[8275, 8272, 8275, 7947, 7944, 8274, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[8273, 8275, 8275, 8273, 7951, 7943, 8272, 8274] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97709 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[8275, 8272, 8275, 7947, 7944, 8274, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[3, 0, 0, 3, 100, 0, 32765, 32665] avg=8192.0000 max=32765 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[8273, 8275, 8275, 8273, 7951, 7943, 8272, 8274] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97709 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[8274, 8273, 8275, 8273, 7940, 8274, 7953, 8274] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97694 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[3, 0, 0, 3, 100, 0, 32765, 32665] avg=8192.0000 max=32765 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[8273, 8275, 8275, 8273, 7951, 7943, 8272, 8274] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97709 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[27, 32755, 3, 0, 3, 21, 34, 32693] avg=8192.0000 max=32755 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[3, 0, 0, 3, 100, 0, 32765, 32665] avg=8192.0000 max=32765 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[27, 32755, 3, 0, 3, 21, 34, 32693] avg=8192.0000 max=32755 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[3, 0, 0, 3, 100, 0, 32765, 32665] avg=8192.0000 max=32765 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[8273, 8275, 8275, 8273, 7951, 7943, 8272, 8274] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97709 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[27, 32755, 3, 0, 3, 21, 34, 32693] avg=8192.0000 max=32755 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[0, 0, 32582, 4, 32764, 183, 3, 0] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[8274, 8273, 8275, 8273, 7940, 8274, 7953, 8274] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97694 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[8273, 8275, 8275, 8273, 7951, 7943, 8272, 8274] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97709 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[8274, 8273, 8275, 8273, 7940, 8274, 7953, 8274] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97694 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[8273, 8275, 8275, 8273, 7951, 7943, 8272, 8274] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97709 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[27, 32755, 3, 0, 3, 21, 34, 32693] avg=8192.0000 max=32755 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[8274, 8273, 8275, 8273, 7940, 8274, 7953, 8274] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97694 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[8276, 8275, 8274, 8275, 8272, 7949, 8274, 7941] avg=8192.0000 max=8276 max_rank=0 min=7941 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97656 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[27, 32755, 3, 0, 3, 21, 34, 32693] avg=8192.0000 max=32755 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[8274, 8273, 8275, 8273, 7940, 8274, 7953, 8274] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97694 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[0, 0, 32582, 4, 32764, 183, 3, 0] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[27, 32755, 3, 0, 3, 21, 34, 32693] avg=8192.0000 max=32755 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[8274, 8273, 8275, 8273, 7940, 8274, 7953, 8274] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97694 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[0, 0, 32582, 4, 32764, 183, 3, 0] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[27, 32755, 3, 0, 3, 21, 34, 32693] avg=8192.0000 max=32755 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[32764, 0, 0, 4, 0, 32764, 3, 1] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[0, 0, 32582, 4, 32764, 183, 3, 0] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[8276, 8275, 8274, 8275, 8272, 7949, 8274, 7941] avg=8192.0000 max=8276 max_rank=0 min=7941 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97656 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[8274, 8273, 8275, 8273, 7940, 8274, 7953, 8274] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97694 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[0, 0, 32582, 4, 32764, 183, 3, 0] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[8276, 8275, 8274, 8275, 8272, 7949, 8274, 7941] avg=8192.0000 max=8276 max_rank=0 min=7941 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97656 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[8274, 8273, 8275, 8273, 7940, 8274, 7953, 8274] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97694 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[8276, 8275, 8274, 8275, 8272, 7949, 8274, 7941] avg=8192.0000 max=8276 max_rank=0 min=7941 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97656 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[0, 0, 32582, 4, 32764, 183, 3, 0] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[8276, 8275, 8274, 8275, 8272, 7949, 8274, 7941] avg=8192.0000 max=8276 max_rank=0 min=7941 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97656 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[32764, 0, 0, 4, 0, 32764, 3, 1] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[0, 0, 32582, 4, 32764, 183, 3, 0] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[8276, 8275, 8274, 8275, 8272, 7949, 8274, 7941] avg=8192.0000 max=8276 max_rank=0 min=7941 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97656 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[32764, 0, 0, 4, 0, 32764, 3, 1] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[0, 0, 32582, 4, 32764, 183, 3, 0] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[3, 1, 32764, 0, 0, 3, 0, 32765] avg=8192.0000 max=32765 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[32764, 0, 0, 4, 0, 32764, 3, 1] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[8276, 8275, 8274, 8275, 8272, 7949, 8274, 7941] avg=8192.0000 max=8276 max_rank=0 min=7941 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97656 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[32764, 0, 0, 4, 0, 32764, 3, 1] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[8276, 8275, 8274, 8275, 8272, 7949, 8274, 7941] avg=8192.0000 max=8276 max_rank=0 min=7941 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97656 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[32764, 0, 0, 4, 0, 32764, 3, 1] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[3, 1, 32764, 0, 0, 3, 0, 32765] avg=8192.0000 max=32765 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[32764, 0, 0, 4, 0, 32764, 3, 1] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[3, 1, 32764, 0, 0, 3, 0, 32765] avg=8192.0000 max=32765 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[32764, 0, 0, 4, 0, 32764, 3, 1] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[0, 32764, 32764, 0, 0, 4, 4, 0] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[3, 1, 32764, 0, 0, 3, 0, 32765] avg=8192.0000 max=32765 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[3, 1, 32764, 0, 0, 3, 0, 32765] avg=8192.0000 max=32765 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[3, 1, 32764, 0, 0, 3, 0, 32765] avg=8192.0000 max=32765 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[8275, 8275, 8276, 8274, 7940, 8275, 7948, 8273] avg=8192.0000 max=8276 max_rank=2 min=7940 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[0, 32764, 32764, 0, 0, 4, 4, 0] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[3, 1, 32764, 0, 0, 3, 0, 32765] avg=8192.0000 max=32765 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[0, 32764, 32764, 0, 0, 4, 4, 0] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[3, 1, 32764, 0, 0, 3, 0, 32765] avg=8192.0000 max=32765 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[0, 32764, 32764, 0, 0, 4, 4, 0] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[8275, 8275, 8276, 8274, 7940, 8275, 7948, 8273] avg=8192.0000 max=8276 max_rank=2 min=7940 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[0, 32764, 32764, 0, 0, 4, 4, 0] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[0, 32764, 32764, 0, 0, 4, 4, 0] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[8275, 8275, 8276, 8274, 7940, 8275, 7948, 8273] avg=8192.0000 max=8276 max_rank=2 min=7940 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[8275, 8275, 8276, 8274, 7940, 8275, 7948, 8273] avg=8192.0000 max=8276 max_rank=2 min=7940 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[8275, 8275, 8276, 8274, 7940, 8275, 7948, 8273] avg=8192.0000 max=8276 max_rank=2 min=7940 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[0, 32764, 32764, 0, 0, 4, 4, 0] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[8275, 8275, 8276, 8274, 7940, 8275, 7948, 8273] avg=8192.0000 max=8276 max_rank=2 min=7940 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[0, 32764, 32764, 0, 0, 4, 4, 0] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[8275, 8275, 8276, 8274, 7940, 8275, 7948, 8273] avg=8192.0000 max=8276 max_rank=2 min=7940 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[8275, 8275, 8276, 8274, 7940, 8275, 7948, 8273] avg=8192.0000 max=8276 max_rank=2 min=7940 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=32 total_expected_recv=257897 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=65664 peak_demand=8276 required_capacity=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=32 total_expected_recv=264751 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=59520 peak_demand=8930 required_capacity=1.090088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=32 total_expected_recv=264385 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=59648 peak_demand=8930 required_capacity=1.090088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=32 total_expected_recv=264466 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=59392 peak_demand=8930 required_capacity=1.090088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=32 total_expected_recv=258451 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=65664 peak_demand=8275 required_capacity=1.010132 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=32 total_expected_recv=258636 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=64896 peak_demand=8275 required_capacity=1.010132 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=32 total_expected_recv=263135 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=60416 peak_demand=8930 required_capacity=1.090088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=32 total_expected_recv=265431 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=59264 peak_demand=8930 required_capacity=1.090088 + [2026-08-30 05:08:02] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 900.1 | throughput per GPU (TFLOP/s/GPU): 370.4 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.892546E+00 | loss scale: 1.0 | grad norm: 7.933 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 10240 | min safe capacity: 1.2500 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=10240 expected_recv=22221 expected_overflow=2187 exact_recv_per_rank=[12427, 7463, 2432, 12427, 3608, 12428, 12426, 2325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=10240 expected_recv=8286 expected_overflow=0 exact_recv_per_rank=[12427, 7463, 2432, 12427, 3608, 12428, 12426, 2325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=10240 expected_recv=12702 expected_overflow=0 exact_recv_per_rank=[12427, 7463, 2432, 12427, 3608, 12428, 12426, 2325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=10240 expected_recv=22221 expected_overflow=2187 exact_recv_per_rank=[12427, 7463, 2432, 12427, 3608, 12428, 12426, 2325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=10240 expected_recv=22223 expected_overflow=2188 exact_recv_per_rank=[12427, 7463, 2432, 12427, 3608, 12428, 12426, 2325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=10240 expected_recv=22220 expected_overflow=2186 exact_recv_per_rank=[12427, 7463, 2432, 12427, 3608, 12428, 12426, 2325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=10240 expected_recv=12121 expected_overflow=0 exact_recv_per_rank=[12427, 7463, 2432, 12427, 3608, 12428, 12426, 2325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=10240 expected_recv=22221 actual_valid_recv=20096 observed_loss=2187 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=10240 expected_recv=8286 actual_valid_recv=8448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=10240 expected_recv=9078 expected_overflow=0 exact_recv_per_rank=[12427, 7463, 2432, 12427, 3608, 12428, 12426, 2325] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=10240 expected_recv=12702 actual_valid_recv=13056 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=10240 expected_recv=22221 actual_valid_recv=20096 observed_loss=2187 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=10240 expected_recv=22223 actual_valid_recv=20096 observed_loss=2188 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=20096 slack_slots=384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=8448 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=10240 expected_recv=22220 actual_valid_recv=20224 observed_loss=2186 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=10240 expected_recv=12121 actual_valid_recv=12544 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=13056 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=10240 expected_recv=9078 actual_valid_recv=9216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=20096 slack_slots=384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=20096 slack_slots=384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=20224 slack_slots=256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=12544 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=9216 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=10240 expected_recv=16221 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8275, 8274, 8275, 7947, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8275, 8274, 8275, 7947, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8275, 8274, 8275, 7947, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8275, 8274, 8275, 7947, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=10240 expected_recv=16221 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=10240 expected_recv=16222 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8275, 8274, 8275, 7947, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=10240 expected_recv=15887 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8275, 8274, 8275, 7947, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8275, 8274, 8275, 7947, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8275, 8274, 8275, 7947, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=10240 expected_recv=16222 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=10240 expected_recv=15887 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=10240 expected_recv=15896 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 7950, 8275, 8274, 8275, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 7950, 8275, 8274, 8275, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 7950, 8275, 8274, 8275, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=10240 expected_recv=15896 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 7950, 8275, 8274, 8275, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 7950, 8275, 8274, 8275, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=10240 expected_recv=16215 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 7950, 8275, 8274, 8275, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 7950, 8275, 8274, 8275, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 7950, 8275, 8274, 8275, 7941] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=10240 expected_recv=16215 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=10240 expected_recv=22992 expected_overflow=2770 exact_recv_per_rank=[9982, 9981, 9982, 9982, 1619, 9982, 9981, 4027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=10240 expected_recv=22990 expected_overflow=2769 exact_recv_per_rank=[9982, 9981, 9982, 9982, 1619, 9982, 9981, 4027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=10240 expected_recv=22992 actual_valid_recv=20224 observed_loss=2770 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=10240 expected_recv=15673 expected_overflow=0 exact_recv_per_rank=[9982, 9981, 9982, 9982, 1619, 9982, 9981, 4027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=10240 expected_recv=22991 expected_overflow=2769 exact_recv_per_rank=[9982, 9981, 9982, 9982, 1619, 9982, 9981, 4027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=10240 expected_recv=9986 expected_overflow=0 exact_recv_per_rank=[9982, 9981, 9982, 9982, 1619, 9982, 9981, 4027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=10240 expected_recv=4028 expected_overflow=0 exact_recv_per_rank=[9982, 9981, 9982, 9982, 1619, 9982, 9981, 4027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=20224 slack_slots=256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=10240 expected_recv=22990 actual_valid_recv=20224 observed_loss=2769 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=10240 expected_recv=15673 actual_valid_recv=16000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=10240 expected_recv=22991 actual_valid_recv=20352 observed_loss=2769 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=10240 expected_recv=9986 actual_valid_recv=10240 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=10240 expected_recv=4028 actual_valid_recv=4352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=10240 expected_recv=22990 expected_overflow=2769 exact_recv_per_rank=[9982, 9981, 9982, 9982, 1619, 9982, 9981, 4027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=20224 slack_slots=256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=20352 slack_slots=128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16000 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=10240 slack_slots=10240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=10240 expected_recv=9422 expected_overflow=0 exact_recv_per_rank=[9982, 9981, 9982, 9982, 1619, 9982, 9981, 4027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=4352 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=10240 expected_recv=22990 actual_valid_recv=20224 observed_loss=2769 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=10240 expected_recv=9422 actual_valid_recv=9728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=20224 slack_slots=256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 8275, 7940, 8273, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=9728 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 8275, 7940, 8273, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 8275, 7940, 8273, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 8275, 7940, 8273, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=10240 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 8275, 7940, 8273, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=10240 expected_recv=16226 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 8275, 7940, 8273, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 8275, 7940, 8273, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=10240 expected_recv=16216 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=10240 expected_recv=16226 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=10240 expected_recv=16212 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 8275, 7940, 8273, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=10240 expected_recv=16212 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 7947, 8275, 7945, 8275, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 7947, 8275, 7945, 8275, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 7947, 8275, 7945, 8275, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=10240 expected_recv=16223 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 7947, 8275, 7945, 8275, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=10240 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 7947, 8275, 7945, 8275, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 7947, 8275, 7945, 8275, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=10240 expected_recv=16223 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=10240 expected_recv=16545 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 7947, 8275, 7945, 8275, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=10240 expected_recv=15892 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 7947, 8275, 7945, 8275, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=10240 expected_recv=15892 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=10240 expected_recv=16206 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 8274, 7934, 8274, 8274, 8275, 7958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=10240 expected_recv=16206 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 8274, 7934, 8274, 8274, 8275, 7958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=10240 expected_recv=15904 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 8274, 7934, 8274, 8274, 8275, 7958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 8274, 7934, 8274, 8274, 8275, 7958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 8274, 7934, 8274, 8274, 8275, 7958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=10240 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 8274, 7934, 8274, 8274, 8275, 7958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 8274, 7934, 8274, 8274, 8275, 7958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 8274, 7934, 8274, 8274, 8275, 7958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=10240 expected_recv=15904 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=10240 expected_recv=16217 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=10240 expected_recv=21138 expected_overflow=2937 exact_recv_per_rank=[8275, 7930, 8275, 7961, 8272, 8275, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=10240 expected_recv=21138 actual_valid_recv=18304 observed_loss=2937 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=18304 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=10240 expected_recv=21452 expected_overflow=2937 exact_recv_per_rank=[8275, 7930, 8275, 7961, 8272, 8275, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=10240 expected_recv=14686 expected_overflow=0 exact_recv_per_rank=[8275, 7930, 8275, 7961, 8272, 8275, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=10240 expected_recv=21106 expected_overflow=2936 exact_recv_per_rank=[8275, 7930, 8275, 7961, 8272, 8275, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=10240 expected_recv=21452 actual_valid_recv=18560 observed_loss=2937 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=10240 expected_recv=8276 expected_overflow=0 exact_recv_per_rank=[8275, 7930, 8275, 7961, 8272, 8275, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=10240 expected_recv=14686 actual_valid_recv=14848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=10240 expected_recv=8273 expected_overflow=0 exact_recv_per_rank=[8275, 7930, 8275, 7961, 8272, 8275, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=10240 expected_recv=21455 expected_overflow=2940 exact_recv_per_rank=[8275, 7930, 8275, 7961, 8272, 8275, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=10240 expected_recv=14686 expected_overflow=0 exact_recv_per_rank=[8275, 7930, 8275, 7961, 8272, 8275, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=18560 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=10240 expected_recv=21106 actual_valid_recv=18304 observed_loss=2936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=10240 expected_recv=8276 actual_valid_recv=8576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=14848 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=10240 expected_recv=8273 actual_valid_recv=8576 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=10240 expected_recv=21455 actual_valid_recv=18560 observed_loss=2940 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=18304 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=10240 expected_recv=14686 actual_valid_recv=14848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=8576 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=10240 expected_recv=15879 expected_overflow=0 exact_recv_per_rank=[7957, 8275, 8274, 7935, 8274, 8274, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=8576 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=18560 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=14848 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=10240 expected_recv=15879 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=10240 expected_recv=16233 expected_overflow=0 exact_recv_per_rank=[7957, 8275, 8274, 7935, 8274, 8274, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=10240 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[7957, 8275, 8274, 7935, 8274, 8274, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=10240 expected_recv=16233 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=10240 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7957, 8275, 8274, 7935, 8274, 8274, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[7957, 8275, 8274, 7935, 8274, 8274, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7957, 8275, 8274, 7935, 8274, 8274, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=10240 expected_recv=16222 expected_overflow=0 exact_recv_per_rank=[7957, 8275, 8274, 7935, 8274, 8274, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[7957, 8275, 8274, 7935, 8274, 8274, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=10240 expected_recv=16222 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=10240 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7961, 8274, 8273, 8275, 7932, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=10240 expected_recv=16220 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7961, 8274, 8273, 8275, 7932, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7961, 8274, 8273, 8275, 7932, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7961, 8274, 8273, 8275, 7932, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=10240 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7961, 8274, 8273, 8275, 7932, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=10240 expected_recv=16234 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7961, 8274, 8273, 8275, 7932, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7961, 8274, 8273, 8275, 7932, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=10240 expected_recv=16207 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7961, 8274, 8273, 8275, 7932, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=10240 expected_recv=16217 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=10240 expected_recv=11012 expected_overflow=0 exact_recv_per_rank=[10002, 10004, 10005, 2737, 2776, 10004, 10005, 10003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=10240 expected_recv=16234 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=10240 expected_recv=16207 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=10240 expected_recv=11012 actual_valid_recv=11392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=10240 expected_recv=18274 expected_overflow=0 exact_recv_per_rank=[10002, 10004, 10005, 2737, 2776, 10004, 10005, 10003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=10240 expected_recv=18275 expected_overflow=0 exact_recv_per_rank=[10002, 10004, 10005, 2737, 2776, 10004, 10005, 10003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=11392 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=10240 expected_recv=18274 actual_valid_recv=18688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=10240 expected_recv=18275 actual_valid_recv=18432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=18688 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=18432 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=10240 expected_recv=17951 expected_overflow=0 exact_recv_per_rank=[10002, 10004, 10005, 2737, 2776, 10004, 10005, 10003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=10240 expected_recv=11052 expected_overflow=0 exact_recv_per_rank=[10002, 10004, 10005, 2737, 2776, 10004, 10005, 10003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=10240 expected_recv=18281 expected_overflow=0 exact_recv_per_rank=[10002, 10004, 10005, 2737, 2776, 10004, 10005, 10003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=10240 expected_recv=17947 expected_overflow=0 exact_recv_per_rank=[10002, 10004, 10005, 2737, 2776, 10004, 10005, 10003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=10240 expected_recv=18280 expected_overflow=0 exact_recv_per_rank=[10002, 10004, 10005, 2737, 2776, 10004, 10005, 10003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=10240 expected_recv=17951 actual_valid_recv=18176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=10240 expected_recv=11052 actual_valid_recv=11264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=10240 expected_recv=18281 actual_valid_recv=18432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=10240 expected_recv=17947 actual_valid_recv=18304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=10240 expected_recv=18280 actual_valid_recv=18432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=18176 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=11264 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=18432 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 7961, 8274, 8274, 8275, 7931, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=18304 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=18432 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 7961, 8274, 8274, 8275, 7931, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 7961, 8274, 8274, 8275, 7931, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 7961, 8274, 8274, 8275, 7931, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 7961, 8274, 8274, 8275, 7931, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=10240 expected_recv=15907 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 7961, 8274, 8274, 8275, 7931, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=10240 expected_recv=16206 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 7961, 8274, 8274, 8275, 7931, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 7961, 8274, 8274, 8275, 7931, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=10240 expected_recv=15907 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=10240 expected_recv=16206 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8274, 8275, 8273, 8275, 7928, 7961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=10240 expected_recv=15906 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8274, 8275, 8273, 8275, 7928, 7961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8274, 8275, 8273, 8275, 7928, 7961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=10240 expected_recv=15906 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8274, 8275, 8273, 8275, 7928, 7961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=10240 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8274, 8275, 8273, 8275, 7928, 7961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=10240 expected_recv=16204 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8274, 8275, 8273, 8275, 7928, 7961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8274, 8275, 8273, 8275, 7928, 7961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8274, 8275, 8273, 8275, 7928, 7961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=10240 expected_recv=16217 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=10240 expected_recv=16204 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=10240 expected_recv=15908 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 8274, 7962, 8273, 8275, 7931, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 8274, 7962, 8273, 8275, 7931, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 8274, 7962, 8273, 8275, 7931, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=10240 expected_recv=15908 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 8274, 7962, 8273, 8275, 7931, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=10240 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 8274, 7962, 8273, 8275, 7931, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=10240 expected_recv=15875 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 8274, 7962, 8273, 8275, 7931, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 8274, 7962, 8273, 8275, 7931, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 8274, 7962, 8273, 8275, 7931, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=10240 expected_recv=16545 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=10240 expected_recv=15875 actual_valid_recv=16000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[7961, 7929, 8275, 8274, 8274, 8273, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7961, 7929, 8275, 8274, 8274, 8273, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16000 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=10240 expected_recv=16237 expected_overflow=0 exact_recv_per_rank=[7961, 7929, 8275, 8274, 8274, 8273, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=10240 expected_recv=16237 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=10240 expected_recv=16202 expected_overflow=0 exact_recv_per_rank=[7961, 7929, 8275, 8274, 8274, 8273, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[7961, 7929, 8275, 8274, 8274, 8273, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[7961, 7929, 8275, 8274, 8274, 8273, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=10240 expected_recv=16202 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7961, 7929, 8275, 8274, 8274, 8273, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=10240 expected_recv=16221 expected_overflow=0 exact_recv_per_rank=[7961, 7929, 8275, 8274, 8274, 8273, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8276, 8273, 7928, 8274, 7964, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8276, 8273, 7928, 8274, 7964, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=10240 expected_recv=16221 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8276, 8273, 7928, 8274, 7964, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8276, 8273, 7928, 8274, 7964, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=10240 expected_recv=16200 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8276, 8273, 7928, 8274, 7964, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=10240 expected_recv=15912 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8276, 8273, 7928, 8274, 7964, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=10240 expected_recv=16200 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=10240 expected_recv=15912 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8276, 8273, 7928, 8274, 7964, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=10240 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8276, 8273, 7928, 8274, 7964, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[19863, 27, 2432, 12427, 3608, 27172, 3, 4] avg=8192.0000 max=27172 max_rank=5 min=3 min_rank=6 imbalance_ratio=3.316895 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[19863, 27, 2432, 12427, 3608, 27172, 3, 4] avg=8192.0000 max=27172 max_rank=5 min=3 min_rank=6 imbalance_ratio=3.316895 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=10240 expected_recv=16217 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[19863, 27, 2432, 12427, 3608, 27172, 3, 4] avg=8192.0000 max=27172 max_rank=5 min=3 min_rank=6 imbalance_ratio=3.316895 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[12427, 7463, 2432, 12427, 3608, 12428, 12426, 2325] avg=8192.0000 max=12428 max_rank=5 min=2325 min_rank=7 imbalance_ratio=1.517090 max_load_reduction=14744 max_load_reduction_pct=0.542617 moved_to_replicas=51262 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[19863, 27, 2432, 12427, 3608, 27172, 3, 4] avg=8192.0000 max=27172 max_rank=5 min=3 min_rank=6 imbalance_ratio=3.316895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[19863, 27, 2432, 12427, 3608, 27172, 3, 4] avg=8192.0000 max=27172 max_rank=5 min=3 min_rank=6 imbalance_ratio=3.316895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[12427, 7463, 2432, 12427, 3608, 12428, 12426, 2325] avg=8192.0000 max=12428 max_rank=5 min=2325 min_rank=7 imbalance_ratio=1.517090 max_load_reduction=14744 max_load_reduction_pct=0.542617 moved_to_replicas=51262 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[19863, 27, 2432, 12427, 3608, 27172, 3, 4] avg=8192.0000 max=27172 max_rank=5 min=3 min_rank=6 imbalance_ratio=3.316895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[12427, 7463, 2432, 12427, 3608, 12428, 12426, 2325] avg=8192.0000 max=12428 max_rank=5 min=2325 min_rank=7 imbalance_ratio=1.517090 max_load_reduction=14744 max_load_reduction_pct=0.542617 moved_to_replicas=51262 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[19863, 27, 2432, 12427, 3608, 27172, 3, 4] avg=8192.0000 max=27172 max_rank=5 min=3 min_rank=6 imbalance_ratio=3.316895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[19863, 27, 2432, 12427, 3608, 27172, 3, 4] avg=8192.0000 max=27172 max_rank=5 min=3 min_rank=6 imbalance_ratio=3.316895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[12427, 7463, 2432, 12427, 3608, 12428, 12426, 2325] avg=8192.0000 max=12428 max_rank=5 min=2325 min_rank=7 imbalance_ratio=1.517090 max_load_reduction=14744 max_load_reduction_pct=0.542617 moved_to_replicas=51262 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[12427, 7463, 2432, 12427, 3608, 12428, 12426, 2325] avg=8192.0000 max=12428 max_rank=5 min=2325 min_rank=7 imbalance_ratio=1.517090 max_load_reduction=14744 max_load_reduction_pct=0.542617 moved_to_replicas=51262 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[12427, 7463, 2432, 12427, 3608, 12428, 12426, 2325] avg=8192.0000 max=12428 max_rank=5 min=2325 min_rank=7 imbalance_ratio=1.517090 max_load_reduction=14744 max_load_reduction_pct=0.542617 moved_to_replicas=51262 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[2, 1, 32760, 6, 32751, 9, 4, 3] avg=8192.0000 max=32760 max_rank=2 min=1 min_rank=1 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[2, 1, 32760, 6, 32751, 9, 4, 3] avg=8192.0000 max=32760 max_rank=2 min=1 min_rank=1 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[2, 1, 32760, 6, 32751, 9, 4, 3] avg=8192.0000 max=32760 max_rank=2 min=1 min_rank=1 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[12427, 7463, 2432, 12427, 3608, 12428, 12426, 2325] avg=8192.0000 max=12428 max_rank=5 min=2325 min_rank=7 imbalance_ratio=1.517090 max_load_reduction=14744 max_load_reduction_pct=0.542617 moved_to_replicas=51262 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[12427, 7463, 2432, 12427, 3608, 12428, 12426, 2325] avg=8192.0000 max=12428 max_rank=5 min=2325 min_rank=7 imbalance_ratio=1.517090 max_load_reduction=14744 max_load_reduction_pct=0.542617 moved_to_replicas=51262 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[8275, 8273, 8275, 8274, 8275, 7947, 8274, 7943] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24485 max_load_reduction_pct=0.747405 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[8275, 8273, 8275, 8274, 8275, 7947, 8274, 7943] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24485 max_load_reduction_pct=0.747405 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[2, 1, 32760, 6, 32751, 9, 4, 3] avg=8192.0000 max=32760 max_rank=2 min=1 min_rank=1 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[2, 1, 32760, 6, 32751, 9, 4, 3] avg=8192.0000 max=32760 max_rank=2 min=1 min_rank=1 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[2, 1, 32760, 6, 32751, 9, 4, 3] avg=8192.0000 max=32760 max_rank=2 min=1 min_rank=1 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[8275, 8273, 8275, 8274, 8275, 7947, 8274, 7943] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24485 max_load_reduction_pct=0.747405 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[2, 1, 32760, 6, 32751, 9, 4, 3] avg=8192.0000 max=32760 max_rank=2 min=1 min_rank=1 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[8275, 8273, 8275, 8274, 8275, 7947, 8274, 7943] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24485 max_load_reduction_pct=0.747405 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[10, 2, 32757, 242, 32513, 1, 8, 3] avg=8192.0000 max=32757 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[2, 1, 32760, 6, 32751, 9, 4, 3] avg=8192.0000 max=32760 max_rank=2 min=1 min_rank=1 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[8275, 8273, 8275, 8274, 8275, 7947, 8274, 7943] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24485 max_load_reduction_pct=0.747405 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[10, 2, 32757, 242, 32513, 1, 8, 3] avg=8192.0000 max=32757 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[8275, 8273, 8275, 8274, 8275, 7947, 8274, 7943] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24485 max_load_reduction_pct=0.747405 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[8275, 8273, 8275, 8274, 8275, 7947, 8274, 7943] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24485 max_load_reduction_pct=0.747405 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[10, 2, 32757, 242, 32513, 1, 8, 3] avg=8192.0000 max=32757 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[8273, 8274, 8274, 7950, 8275, 8274, 8275, 7941] avg=8192.0000 max=8275 max_rank=4 min=7941 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24482 max_load_reduction_pct=0.747382 moved_to_replicas=97693 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[8275, 8273, 8275, 8274, 8275, 7947, 8274, 7943] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24485 max_load_reduction_pct=0.747405 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[8273, 8274, 8274, 7950, 8275, 8274, 8275, 7941] avg=8192.0000 max=8275 max_rank=4 min=7941 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24482 max_load_reduction_pct=0.747382 moved_to_replicas=97693 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[10, 2, 32757, 242, 32513, 1, 8, 3] avg=8192.0000 max=32757 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[10, 2, 32757, 242, 32513, 1, 8, 3] avg=8192.0000 max=32757 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[10, 2, 32757, 242, 32513, 1, 8, 3] avg=8192.0000 max=32757 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[8273, 8274, 8274, 7950, 8275, 8274, 8275, 7941] avg=8192.0000 max=8275 max_rank=4 min=7941 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24482 max_load_reduction_pct=0.747382 moved_to_replicas=97693 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[10, 2, 32757, 242, 32513, 1, 8, 3] avg=8192.0000 max=32757 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[9, 19954, 5, 31547, 7, 6, 13997, 11] avg=8192.0000 max=31547 max_rank=3 min=5 min_rank=2 imbalance_ratio=3.850952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[10, 2, 32757, 242, 32513, 1, 8, 3] avg=8192.0000 max=32757 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[8273, 8274, 8274, 7950, 8275, 8274, 8275, 7941] avg=8192.0000 max=8275 max_rank=4 min=7941 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24482 max_load_reduction_pct=0.747382 moved_to_replicas=97693 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[8273, 8274, 8274, 7950, 8275, 8274, 8275, 7941] avg=8192.0000 max=8275 max_rank=4 min=7941 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24482 max_load_reduction_pct=0.747382 moved_to_replicas=97693 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[8273, 8274, 8274, 7950, 8275, 8274, 8275, 7941] avg=8192.0000 max=8275 max_rank=4 min=7941 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24482 max_load_reduction_pct=0.747382 moved_to_replicas=97693 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[9, 19954, 5, 31547, 7, 6, 13997, 11] avg=8192.0000 max=31547 max_rank=3 min=5 min_rank=2 imbalance_ratio=3.850952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[8273, 8274, 8274, 7950, 8275, 8274, 8275, 7941] avg=8192.0000 max=8275 max_rank=4 min=7941 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24482 max_load_reduction_pct=0.747382 moved_to_replicas=97693 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[9982, 9981, 9982, 9982, 1619, 9982, 9981, 4027] avg=8192.0000 max=9982 max_rank=0 min=1619 min_rank=4 imbalance_ratio=1.218506 max_load_reduction=21565 max_load_reduction_pct=0.683583 moved_to_replicas=62054 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[9, 19954, 5, 31547, 7, 6, 13997, 11] avg=8192.0000 max=31547 max_rank=3 min=5 min_rank=2 imbalance_ratio=3.850952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[8273, 8274, 8274, 7950, 8275, 8274, 8275, 7941] avg=8192.0000 max=8275 max_rank=4 min=7941 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24482 max_load_reduction_pct=0.747382 moved_to_replicas=97693 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[9982, 9981, 9982, 9982, 1619, 9982, 9981, 4027] avg=8192.0000 max=9982 max_rank=0 min=1619 min_rank=4 imbalance_ratio=1.218506 max_load_reduction=21565 max_load_reduction_pct=0.683583 moved_to_replicas=62054 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[9, 19954, 5, 31547, 7, 6, 13997, 11] avg=8192.0000 max=31547 max_rank=3 min=5 min_rank=2 imbalance_ratio=3.850952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[9, 19954, 5, 31547, 7, 6, 13997, 11] avg=8192.0000 max=31547 max_rank=3 min=5 min_rank=2 imbalance_ratio=3.850952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[9, 19954, 5, 31547, 7, 6, 13997, 11] avg=8192.0000 max=31547 max_rank=3 min=5 min_rank=2 imbalance_ratio=3.850952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[9982, 9981, 9982, 9982, 1619, 9982, 9981, 4027] avg=8192.0000 max=9982 max_rank=0 min=1619 min_rank=4 imbalance_ratio=1.218506 max_load_reduction=21565 max_load_reduction_pct=0.683583 moved_to_replicas=62054 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[9, 19954, 5, 31547, 7, 6, 13997, 11] avg=8192.0000 max=31547 max_rank=3 min=5 min_rank=2 imbalance_ratio=3.850952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[7, 32755, 0, 32754, 6, 1, 6, 7] avg=8192.0000 max=32755 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[9, 19954, 5, 31547, 7, 6, 13997, 11] avg=8192.0000 max=31547 max_rank=3 min=5 min_rank=2 imbalance_ratio=3.850952 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[9982, 9981, 9982, 9982, 1619, 9982, 9981, 4027] avg=8192.0000 max=9982 max_rank=0 min=1619 min_rank=4 imbalance_ratio=1.218506 max_load_reduction=21565 max_load_reduction_pct=0.683583 moved_to_replicas=62054 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[9982, 9981, 9982, 9982, 1619, 9982, 9981, 4027] avg=8192.0000 max=9982 max_rank=0 min=1619 min_rank=4 imbalance_ratio=1.218506 max_load_reduction=21565 max_load_reduction_pct=0.683583 moved_to_replicas=62054 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[9982, 9981, 9982, 9982, 1619, 9982, 9981, 4027] avg=8192.0000 max=9982 max_rank=0 min=1619 min_rank=4 imbalance_ratio=1.218506 max_load_reduction=21565 max_load_reduction_pct=0.683583 moved_to_replicas=62054 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[7, 32755, 0, 32754, 6, 1, 6, 7] avg=8192.0000 max=32755 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[9982, 9981, 9982, 9982, 1619, 9982, 9981, 4027] avg=8192.0000 max=9982 max_rank=0 min=1619 min_rank=4 imbalance_ratio=1.218506 max_load_reduction=21565 max_load_reduction_pct=0.683583 moved_to_replicas=62054 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[8275, 8275, 8274, 8275, 7940, 8273, 8273, 7951] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[7, 32755, 0, 32754, 6, 1, 6, 7] avg=8192.0000 max=32755 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[9982, 9981, 9982, 9982, 1619, 9982, 9981, 4027] avg=8192.0000 max=9982 max_rank=0 min=1619 min_rank=4 imbalance_ratio=1.218506 max_load_reduction=21565 max_load_reduction_pct=0.683583 moved_to_replicas=62054 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[8275, 8275, 8274, 8275, 7940, 8273, 8273, 7951] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[7, 32755, 0, 32754, 6, 1, 6, 7] avg=8192.0000 max=32755 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[7, 32755, 0, 32754, 6, 1, 6, 7] avg=8192.0000 max=32755 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[7, 32755, 0, 32754, 6, 1, 6, 7] avg=8192.0000 max=32755 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[8275, 8275, 8274, 8275, 7940, 8273, 8273, 7951] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[7, 32755, 0, 32754, 6, 1, 6, 7] avg=8192.0000 max=32755 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[30684, 13, 7, 4, 2061, 32754, 8, 5] avg=8192.0000 max=32754 max_rank=5 min=4 min_rank=3 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[7, 32755, 0, 32754, 6, 1, 6, 7] avg=8192.0000 max=32755 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[8275, 8275, 8274, 8275, 7940, 8273, 8273, 7951] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[8275, 8275, 8274, 8275, 7940, 8273, 8273, 7951] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[30684, 13, 7, 4, 2061, 32754, 8, 5] avg=8192.0000 max=32754 max_rank=5 min=4 min_rank=3 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[8275, 8275, 8274, 8275, 7940, 8273, 8273, 7951] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[8275, 8275, 8274, 8275, 7940, 8273, 8273, 7951] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[8274, 8274, 7947, 8275, 7945, 8275, 8273, 8273] avg=8192.0000 max=8275 max_rank=3 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=95870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[30684, 13, 7, 4, 2061, 32754, 8, 5] avg=8192.0000 max=32754 max_rank=5 min=4 min_rank=3 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[8275, 8275, 8274, 8275, 7940, 8273, 8273, 7951] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[8274, 8274, 7947, 8275, 7945, 8275, 8273, 8273] avg=8192.0000 max=8275 max_rank=3 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=95870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[30684, 13, 7, 4, 2061, 32754, 8, 5] avg=8192.0000 max=32754 max_rank=5 min=4 min_rank=3 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[30684, 13, 7, 4, 2061, 32754, 8, 5] avg=8192.0000 max=32754 max_rank=5 min=4 min_rank=3 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[30684, 13, 7, 4, 2061, 32754, 8, 5] avg=8192.0000 max=32754 max_rank=5 min=4 min_rank=3 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[8274, 8274, 7947, 8275, 7945, 8275, 8273, 8273] avg=8192.0000 max=8275 max_rank=3 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=95870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[0, 32753, 9, 2, 32752, 0, 7, 13] avg=8192.0000 max=32753 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[30684, 13, 7, 4, 2061, 32754, 8, 5] avg=8192.0000 max=32754 max_rank=5 min=4 min_rank=3 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[30684, 13, 7, 4, 2061, 32754, 8, 5] avg=8192.0000 max=32754 max_rank=5 min=4 min_rank=3 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[8274, 8274, 7947, 8275, 7945, 8275, 8273, 8273] avg=8192.0000 max=8275 max_rank=3 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=95870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[0, 32753, 9, 2, 32752, 0, 7, 13] avg=8192.0000 max=32753 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[8274, 8274, 7947, 8275, 7945, 8275, 8273, 8273] avg=8192.0000 max=8275 max_rank=3 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=95870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[8274, 8274, 7947, 8275, 7945, 8275, 8273, 8273] avg=8192.0000 max=8275 max_rank=3 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=95870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[8272, 8275, 8274, 7934, 8274, 8274, 8275, 7958] avg=8192.0000 max=8275 max_rank=1 min=7934 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97943 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[8274, 8274, 7947, 8275, 7945, 8275, 8273, 8273] avg=8192.0000 max=8275 max_rank=3 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=95870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[8274, 8274, 7947, 8275, 7945, 8275, 8273, 8273] avg=8192.0000 max=8275 max_rank=3 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=95870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[0, 32753, 9, 2, 32752, 0, 7, 13] avg=8192.0000 max=32753 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[8272, 8275, 8274, 7934, 8274, 8274, 8275, 7958] avg=8192.0000 max=8275 max_rank=1 min=7934 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97943 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[0, 32753, 9, 2, 32752, 0, 7, 13] avg=8192.0000 max=32753 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[0, 32753, 9, 2, 32752, 0, 7, 13] avg=8192.0000 max=32753 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[25098, 1, 14, 7656, 15, 32752, 0, 0] avg=8192.0000 max=32752 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[0, 32753, 9, 2, 32752, 0, 7, 13] avg=8192.0000 max=32753 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[8272, 8275, 8274, 7934, 8274, 8274, 8275, 7958] avg=8192.0000 max=8275 max_rank=1 min=7934 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97943 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[0, 32753, 9, 2, 32752, 0, 7, 13] avg=8192.0000 max=32753 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[0, 32753, 9, 2, 32752, 0, 7, 13] avg=8192.0000 max=32753 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[25098, 1, 14, 7656, 15, 32752, 0, 0] avg=8192.0000 max=32752 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[8272, 8275, 8274, 7934, 8274, 8274, 8275, 7958] avg=8192.0000 max=8275 max_rank=1 min=7934 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97943 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[8275, 7930, 8275, 7961, 8272, 8275, 8275, 8273] avg=8192.0000 max=8275 max_rank=0 min=7930 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=67305 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[8272, 8275, 8274, 7934, 8274, 8274, 8275, 7958] avg=8192.0000 max=8275 max_rank=1 min=7934 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97943 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[8272, 8275, 8274, 7934, 8274, 8274, 8275, 7958] avg=8192.0000 max=8275 max_rank=1 min=7934 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97943 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[8272, 8275, 8274, 7934, 8274, 8274, 8275, 7958] avg=8192.0000 max=8275 max_rank=1 min=7934 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97943 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[8272, 8275, 8274, 7934, 8274, 8274, 8275, 7958] avg=8192.0000 max=8275 max_rank=1 min=7934 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97943 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[25098, 1, 14, 7656, 15, 32752, 0, 0] avg=8192.0000 max=32752 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[8275, 7930, 8275, 7961, 8272, 8275, 8275, 8273] avg=8192.0000 max=8275 max_rank=0 min=7930 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=67305 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[25098, 1, 14, 7656, 15, 32752, 0, 0] avg=8192.0000 max=32752 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[12, 32750, 0, 3, 11, 7, 0, 32753] avg=8192.0000 max=32753 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[25098, 1, 14, 7656, 15, 32752, 0, 0] avg=8192.0000 max=32752 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[25098, 1, 14, 7656, 15, 32752, 0, 0] avg=8192.0000 max=32752 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[25098, 1, 14, 7656, 15, 32752, 0, 0] avg=8192.0000 max=32752 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[8275, 7930, 8275, 7961, 8272, 8275, 8275, 8273] avg=8192.0000 max=8275 max_rank=0 min=7930 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=67305 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[25098, 1, 14, 7656, 15, 32752, 0, 0] avg=8192.0000 max=32752 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[12, 32750, 0, 3, 11, 7, 0, 32753] avg=8192.0000 max=32753 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[8275, 7930, 8275, 7961, 8272, 8275, 8275, 8273] avg=8192.0000 max=8275 max_rank=0 min=7930 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=67305 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[7957, 8275, 8274, 7935, 8274, 8274, 8275, 8272] avg=8192.0000 max=8275 max_rank=1 min=7935 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97941 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[8275, 7930, 8275, 7961, 8272, 8275, 8275, 8273] avg=8192.0000 max=8275 max_rank=0 min=7930 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=67305 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[8275, 7930, 8275, 7961, 8272, 8275, 8275, 8273] avg=8192.0000 max=8275 max_rank=0 min=7930 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=67305 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[8275, 7930, 8275, 7961, 8272, 8275, 8275, 8273] avg=8192.0000 max=8275 max_rank=0 min=7930 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=67305 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[8275, 7930, 8275, 7961, 8272, 8275, 8275, 8273] avg=8192.0000 max=8275 max_rank=0 min=7930 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=67305 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[12, 32750, 0, 3, 11, 7, 0, 32753] avg=8192.0000 max=32753 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[7957, 8275, 8274, 7935, 8274, 8274, 8275, 8272] avg=8192.0000 max=8275 max_rank=1 min=7935 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97941 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[12, 32750, 0, 3, 11, 7, 0, 32753] avg=8192.0000 max=32753 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[0, 3, 15, 14, 0, 32753, 0, 32751] avg=8192.0000 max=32753 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[12, 32750, 0, 3, 11, 7, 0, 32753] avg=8192.0000 max=32753 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[7957, 8275, 8274, 7935, 8274, 8274, 8275, 8272] avg=8192.0000 max=8275 max_rank=1 min=7935 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97941 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[12, 32750, 0, 3, 11, 7, 0, 32753] avg=8192.0000 max=32753 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[12, 32750, 0, 3, 11, 7, 0, 32753] avg=8192.0000 max=32753 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[12, 32750, 0, 3, 11, 7, 0, 32753] avg=8192.0000 max=32753 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[0, 3, 15, 14, 0, 32753, 0, 32751] avg=8192.0000 max=32753 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[8273, 8274, 7961, 8274, 8273, 8275, 7932, 8274] avg=8192.0000 max=8275 max_rank=5 min=7932 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97942 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[7957, 8275, 8274, 7935, 8274, 8274, 8275, 8272] avg=8192.0000 max=8275 max_rank=1 min=7935 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97941 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[7957, 8275, 8274, 7935, 8274, 8274, 8275, 8272] avg=8192.0000 max=8275 max_rank=1 min=7935 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97941 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[7957, 8275, 8274, 7935, 8274, 8274, 8275, 8272] avg=8192.0000 max=8275 max_rank=1 min=7935 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97941 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[7957, 8275, 8274, 7935, 8274, 8274, 8275, 8272] avg=8192.0000 max=8275 max_rank=1 min=7935 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97941 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[7957, 8275, 8274, 7935, 8274, 8274, 8275, 8272] avg=8192.0000 max=8275 max_rank=1 min=7935 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97941 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[8273, 8274, 7961, 8274, 8273, 8275, 7932, 8274] avg=8192.0000 max=8275 max_rank=5 min=7932 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97942 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[0, 3, 15, 14, 0, 32753, 0, 32751] avg=8192.0000 max=32753 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[17, 12763, 0, 15, 17, 1, 19990, 32733] avg=8192.0000 max=32733 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[0, 3, 15, 14, 0, 32753, 0, 32751] avg=8192.0000 max=32753 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[0, 3, 15, 14, 0, 32753, 0, 32751] avg=8192.0000 max=32753 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[8273, 8274, 7961, 8274, 8273, 8275, 7932, 8274] avg=8192.0000 max=8275 max_rank=5 min=7932 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97942 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[0, 3, 15, 14, 0, 32753, 0, 32751] avg=8192.0000 max=32753 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[0, 3, 15, 14, 0, 32753, 0, 32751] avg=8192.0000 max=32753 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[0, 3, 15, 14, 0, 32753, 0, 32751] avg=8192.0000 max=32753 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[17, 12763, 0, 15, 17, 1, 19990, 32733] avg=8192.0000 max=32733 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[10002, 10004, 10005, 2737, 2776, 10004, 10005, 10003] avg=8192.0000 max=10005 max_rank=2 min=2737 min_rank=3 imbalance_ratio=1.221313 max_load_reduction=22728 max_load_reduction_pct=0.694345 moved_to_replicas=76971 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[8273, 8274, 7961, 8274, 8273, 8275, 7932, 8274] avg=8192.0000 max=8275 max_rank=5 min=7932 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97942 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[8273, 8274, 7961, 8274, 8273, 8275, 7932, 8274] avg=8192.0000 max=8275 max_rank=5 min=7932 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97942 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[8273, 8274, 7961, 8274, 8273, 8275, 7932, 8274] avg=8192.0000 max=8275 max_rank=5 min=7932 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97942 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[8273, 8274, 7961, 8274, 8273, 8275, 7932, 8274] avg=8192.0000 max=8275 max_rank=5 min=7932 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97942 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[8273, 8274, 7961, 8274, 8273, 8275, 7932, 8274] avg=8192.0000 max=8275 max_rank=5 min=7932 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24478 max_load_reduction_pct=0.747351 moved_to_replicas=97942 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[10002, 10004, 10005, 2737, 2776, 10004, 10005, 10003] avg=8192.0000 max=10005 max_rank=2 min=2737 min_rank=3 imbalance_ratio=1.221313 max_load_reduction=22728 max_load_reduction_pct=0.694345 moved_to_replicas=76971 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[17, 12763, 0, 15, 17, 1, 19990, 32733] avg=8192.0000 max=32733 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[0, 11, 16, 0, 5, 32752, 0, 32752] avg=8192.0000 max=32752 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[17, 12763, 0, 15, 17, 1, 19990, 32733] avg=8192.0000 max=32733 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[17, 12763, 0, 15, 17, 1, 19990, 32733] avg=8192.0000 max=32733 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[10002, 10004, 10005, 2737, 2776, 10004, 10005, 10003] avg=8192.0000 max=10005 max_rank=2 min=2737 min_rank=3 imbalance_ratio=1.221313 max_load_reduction=22728 max_load_reduction_pct=0.694345 moved_to_replicas=76971 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[17, 12763, 0, 15, 17, 1, 19990, 32733] avg=8192.0000 max=32733 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[17, 12763, 0, 15, 17, 1, 19990, 32733] avg=8192.0000 max=32733 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[0, 11, 16, 0, 5, 32752, 0, 32752] avg=8192.0000 max=32752 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[17, 12763, 0, 15, 17, 1, 19990, 32733] avg=8192.0000 max=32733 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[8272, 8275, 7961, 8274, 8274, 8275, 7931, 8274] avg=8192.0000 max=8275 max_rank=1 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97942 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[10002, 10004, 10005, 2737, 2776, 10004, 10005, 10003] avg=8192.0000 max=10005 max_rank=2 min=2737 min_rank=3 imbalance_ratio=1.221313 max_load_reduction=22728 max_load_reduction_pct=0.694345 moved_to_replicas=76971 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[10002, 10004, 10005, 2737, 2776, 10004, 10005, 10003] avg=8192.0000 max=10005 max_rank=2 min=2737 min_rank=3 imbalance_ratio=1.221313 max_load_reduction=22728 max_load_reduction_pct=0.694345 moved_to_replicas=76971 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[10002, 10004, 10005, 2737, 2776, 10004, 10005, 10003] avg=8192.0000 max=10005 max_rank=2 min=2737 min_rank=3 imbalance_ratio=1.221313 max_load_reduction=22728 max_load_reduction_pct=0.694345 moved_to_replicas=76971 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[10002, 10004, 10005, 2737, 2776, 10004, 10005, 10003] avg=8192.0000 max=10005 max_rank=2 min=2737 min_rank=3 imbalance_ratio=1.221313 max_load_reduction=22728 max_load_reduction_pct=0.694345 moved_to_replicas=76971 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[8272, 8275, 7961, 8274, 8274, 8275, 7931, 8274] avg=8192.0000 max=8275 max_rank=1 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97942 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[10002, 10004, 10005, 2737, 2776, 10004, 10005, 10003] avg=8192.0000 max=10005 max_rank=2 min=2737 min_rank=3 imbalance_ratio=1.221313 max_load_reduction=22728 max_load_reduction_pct=0.694345 moved_to_replicas=76971 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[0, 11, 16, 0, 5, 32752, 0, 32752] avg=8192.0000 max=32752 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[0, 3, 32751, 16, 0, 32749, 0, 17] avg=8192.0000 max=32751 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.997925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[0, 11, 16, 0, 5, 32752, 0, 32752] avg=8192.0000 max=32752 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[0, 11, 16, 0, 5, 32752, 0, 32752] avg=8192.0000 max=32752 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[8272, 8275, 7961, 8274, 8274, 8275, 7931, 8274] avg=8192.0000 max=8275 max_rank=1 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97942 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[0, 11, 16, 0, 5, 32752, 0, 32752] avg=8192.0000 max=32752 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[0, 11, 16, 0, 5, 32752, 0, 32752] avg=8192.0000 max=32752 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[8276, 8274, 8274, 8275, 8273, 8275, 7928, 7961] avg=8192.0000 max=8276 max_rank=0 min=7928 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24475 max_load_reduction_pct=0.747305 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[0, 3, 32751, 16, 0, 32749, 0, 17] avg=8192.0000 max=32751 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.997925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[0, 11, 16, 0, 5, 32752, 0, 32752] avg=8192.0000 max=32752 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[8272, 8275, 7961, 8274, 8274, 8275, 7931, 8274] avg=8192.0000 max=8275 max_rank=1 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97942 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[8272, 8275, 7961, 8274, 8274, 8275, 7931, 8274] avg=8192.0000 max=8275 max_rank=1 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97942 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[8272, 8275, 7961, 8274, 8274, 8275, 7931, 8274] avg=8192.0000 max=8275 max_rank=1 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97942 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[8272, 8275, 7961, 8274, 8274, 8275, 7931, 8274] avg=8192.0000 max=8275 max_rank=1 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97942 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[8276, 8274, 8274, 8275, 8273, 8275, 7928, 7961] avg=8192.0000 max=8276 max_rank=0 min=7928 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24475 max_load_reduction_pct=0.747305 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[8272, 8275, 7961, 8274, 8274, 8275, 7931, 8274] avg=8192.0000 max=8275 max_rank=1 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97942 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[0, 3, 32751, 16, 0, 32749, 0, 17] avg=8192.0000 max=32751 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.997925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[32752, 32745, 14, 22, 0, 0, 0, 3] avg=8192.0000 max=32752 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[0, 3, 32751, 16, 0, 32749, 0, 17] avg=8192.0000 max=32751 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.997925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[0, 3, 32751, 16, 0, 32749, 0, 17] avg=8192.0000 max=32751 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.997925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[8276, 8274, 8274, 8275, 8273, 8275, 7928, 7961] avg=8192.0000 max=8276 max_rank=0 min=7928 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24475 max_load_reduction_pct=0.747305 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[0, 3, 32751, 16, 0, 32749, 0, 17] avg=8192.0000 max=32751 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.997925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[8273, 8273, 8274, 7962, 8273, 8275, 7931, 8275] avg=8192.0000 max=8275 max_rank=5 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[0, 3, 32751, 16, 0, 32749, 0, 17] avg=8192.0000 max=32751 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.997925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[32752, 32745, 14, 22, 0, 0, 0, 3] avg=8192.0000 max=32752 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[0, 3, 32751, 16, 0, 32749, 0, 17] avg=8192.0000 max=32751 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.997925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[8276, 8274, 8274, 8275, 8273, 8275, 7928, 7961] avg=8192.0000 max=8276 max_rank=0 min=7928 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24475 max_load_reduction_pct=0.747305 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[8276, 8274, 8274, 8275, 8273, 8275, 7928, 7961] avg=8192.0000 max=8276 max_rank=0 min=7928 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24475 max_load_reduction_pct=0.747305 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[8276, 8274, 8274, 8275, 8273, 8275, 7928, 7961] avg=8192.0000 max=8276 max_rank=0 min=7928 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24475 max_load_reduction_pct=0.747305 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[8276, 8274, 8274, 8275, 8273, 8275, 7928, 7961] avg=8192.0000 max=8276 max_rank=0 min=7928 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24475 max_load_reduction_pct=0.747305 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[8273, 8273, 8274, 7962, 8273, 8275, 7931, 8275] avg=8192.0000 max=8275 max_rank=5 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[32752, 32745, 14, 22, 0, 0, 0, 3] avg=8192.0000 max=32752 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[8276, 8274, 8274, 8275, 8273, 8275, 7928, 7961] avg=8192.0000 max=8276 max_rank=0 min=7928 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24475 max_load_reduction_pct=0.747305 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[15, 3, 32748, 2, 7, 14, 0, 32747] avg=8192.0000 max=32748 max_rank=2 min=0 min_rank=6 imbalance_ratio=3.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[32752, 32745, 14, 22, 0, 0, 0, 3] avg=8192.0000 max=32752 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[32752, 32745, 14, 22, 0, 0, 0, 3] avg=8192.0000 max=32752 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[8273, 8273, 8274, 7962, 8273, 8275, 7931, 8275] avg=8192.0000 max=8275 max_rank=5 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[7961, 7929, 8275, 8274, 8274, 8273, 8275, 8275] avg=8192.0000 max=8275 max_rank=2 min=7929 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24473 max_load_reduction_pct=0.747313 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[32752, 32745, 14, 22, 0, 0, 0, 3] avg=8192.0000 max=32752 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[15, 3, 32748, 2, 7, 14, 0, 32747] avg=8192.0000 max=32748 max_rank=2 min=0 min_rank=6 imbalance_ratio=3.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[32752, 32745, 14, 22, 0, 0, 0, 3] avg=8192.0000 max=32752 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[32752, 32745, 14, 22, 0, 0, 0, 3] avg=8192.0000 max=32752 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[8273, 8273, 8274, 7962, 8273, 8275, 7931, 8275] avg=8192.0000 max=8275 max_rank=5 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[8273, 8273, 8274, 7962, 8273, 8275, 7931, 8275] avg=8192.0000 max=8275 max_rank=5 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[8273, 8273, 8274, 7962, 8273, 8275, 7931, 8275] avg=8192.0000 max=8275 max_rank=5 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[7961, 7929, 8275, 8274, 8274, 8273, 8275, 8275] avg=8192.0000 max=8275 max_rank=2 min=7929 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24473 max_load_reduction_pct=0.747313 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[8273, 8273, 8274, 7962, 8273, 8275, 7931, 8275] avg=8192.0000 max=8275 max_rank=5 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[15, 3, 32748, 2, 7, 14, 0, 32747] avg=8192.0000 max=32748 max_rank=2 min=0 min_rank=6 imbalance_ratio=3.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[8273, 8273, 8274, 7962, 8273, 8275, 7931, 8275] avg=8192.0000 max=8275 max_rank=5 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[17, 32749, 0, 0, 0, 3, 17, 32750] avg=8192.0000 max=32750 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[15, 3, 32748, 2, 7, 14, 0, 32747] avg=8192.0000 max=32748 max_rank=2 min=0 min_rank=6 imbalance_ratio=3.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[15, 3, 32748, 2, 7, 14, 0, 32747] avg=8192.0000 max=32748 max_rank=2 min=0 min_rank=6 imbalance_ratio=3.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[7961, 7929, 8275, 8274, 8274, 8273, 8275, 8275] avg=8192.0000 max=8275 max_rank=2 min=7929 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24473 max_load_reduction_pct=0.747313 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[8273, 8275, 8276, 8273, 7928, 8274, 7964, 8273] avg=8192.0000 max=8276 max_rank=2 min=7928 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24474 max_load_reduction_pct=0.747298 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[15, 3, 32748, 2, 7, 14, 0, 32747] avg=8192.0000 max=32748 max_rank=2 min=0 min_rank=6 imbalance_ratio=3.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[17, 32749, 0, 0, 0, 3, 17, 32750] avg=8192.0000 max=32750 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[15, 3, 32748, 2, 7, 14, 0, 32747] avg=8192.0000 max=32748 max_rank=2 min=0 min_rank=6 imbalance_ratio=3.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[15, 3, 32748, 2, 7, 14, 0, 32747] avg=8192.0000 max=32748 max_rank=2 min=0 min_rank=6 imbalance_ratio=3.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[7961, 7929, 8275, 8274, 8274, 8273, 8275, 8275] avg=8192.0000 max=8275 max_rank=2 min=7929 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24473 max_load_reduction_pct=0.747313 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[7961, 7929, 8275, 8274, 8274, 8273, 8275, 8275] avg=8192.0000 max=8275 max_rank=2 min=7929 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24473 max_load_reduction_pct=0.747313 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[7961, 7929, 8275, 8274, 8274, 8273, 8275, 8275] avg=8192.0000 max=8275 max_rank=2 min=7929 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24473 max_load_reduction_pct=0.747313 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[8273, 8275, 8276, 8273, 7928, 8274, 7964, 8273] avg=8192.0000 max=8276 max_rank=2 min=7928 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24474 max_load_reduction_pct=0.747298 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[7961, 7929, 8275, 8274, 8274, 8273, 8275, 8275] avg=8192.0000 max=8275 max_rank=2 min=7929 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24473 max_load_reduction_pct=0.747313 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[17, 32749, 0, 0, 0, 3, 17, 32750] avg=8192.0000 max=32750 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[7961, 7929, 8275, 8274, 8274, 8273, 8275, 8275] avg=8192.0000 max=8275 max_rank=2 min=7929 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24473 max_load_reduction_pct=0.747313 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[17, 32749, 0, 0, 0, 3, 17, 32750] avg=8192.0000 max=32750 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[17, 32749, 0, 0, 0, 3, 17, 32750] avg=8192.0000 max=32750 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[17, 32749, 0, 0, 0, 3, 17, 32750] avg=8192.0000 max=32750 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[8273, 8275, 8276, 8273, 7928, 8274, 7964, 8273] avg=8192.0000 max=8276 max_rank=2 min=7928 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24474 max_load_reduction_pct=0.747298 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[17, 32749, 0, 0, 0, 3, 17, 32750] avg=8192.0000 max=32750 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[17, 32749, 0, 0, 0, 3, 17, 32750] avg=8192.0000 max=32750 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[8273, 8275, 8276, 8273, 7928, 8274, 7964, 8273] avg=8192.0000 max=8276 max_rank=2 min=7928 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24474 max_load_reduction_pct=0.747298 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[8273, 8275, 8276, 8273, 7928, 8274, 7964, 8273] avg=8192.0000 max=8276 max_rank=2 min=7928 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24474 max_load_reduction_pct=0.747298 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[8273, 8275, 8276, 8273, 7928, 8274, 7964, 8273] avg=8192.0000 max=8276 max_rank=2 min=7928 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24474 max_load_reduction_pct=0.747298 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[8273, 8275, 8276, 8273, 7928, 8274, 7964, 8273] avg=8192.0000 max=8276 max_rank=2 min=7928 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24474 max_load_reduction_pct=0.747298 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[8273, 8275, 8276, 8273, 7928, 8274, 7964, 8273] avg=8192.0000 max=8276 max_rank=2 min=7928 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24474 max_load_reduction_pct=0.747298 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=32 total_expected_recv=272981 total_dropped_tokens=7894 drop_rate_pct=2.891776 records_with_drop_pct=9.375000 peak_rank_overflow=2937 total_slack_slots=58880 peak_demand=13177 required_capacity=1.608521 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=32 total_expected_recv=245080 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78720 peak_demand=10003 required_capacity=1.221069 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=32 total_expected_recv=282892 total_dropped_tokens=7893 drop_rate_pct=2.790111 records_with_drop_pct=9.375000 peak_rank_overflow=2937 total_slack_slots=49664 peak_demand=13177 required_capacity=1.608521 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=32 total_expected_recv=233739 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=90752 peak_demand=8276 required_capacity=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=32 total_expected_recv=272662 total_dropped_tokens=5705 drop_rate_pct=2.092334 records_with_drop_pct=6.250000 peak_rank_overflow=2936 total_slack_slots=57600 peak_demand=13176 required_capacity=1.608398 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=32 total_expected_recv=274096 total_dropped_tokens=4955 drop_rate_pct=1.807761 records_with_drop_pct=6.250000 peak_rank_overflow=2769 total_slack_slots=55424 peak_demand=13009 required_capacity=1.588013 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=32 total_expected_recv=247808 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76288 peak_demand=10005 required_capacity=1.221313 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=32 total_expected_recv=267894 total_dropped_tokens=5128 drop_rate_pct=1.914190 records_with_drop_pct=6.250000 peak_rank_overflow=2940 total_slack_slots=61568 peak_demand=13180 required_capacity=1.608887 + [2026-08-30 05:08:03] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 897.8 | throughput per GPU (TFLOP/s/GPU): 371.3 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.986233E+00 | loss scale: 1.0 | grad norm: 6.429 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 10240 | min safe capacity: 1.2500 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=10240 expected_recv=21119 expected_overflow=2086 exact_recv_per_rank=[544, 12325, 11109, 12326, 12330, 2768, 1804, 12330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=10240 expected_recv=21118 expected_overflow=2085 exact_recv_per_rank=[544, 12325, 11109, 12326, 12330, 2768, 1804, 12330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=10240 expected_recv=19902 expected_overflow=869 exact_recv_per_rank=[544, 12325, 11109, 12326, 12330, 2768, 1804, 12330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=10240 expected_recv=8673 expected_overflow=0 exact_recv_per_rank=[544, 12325, 11109, 12326, 12330, 2768, 1804, 12330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=10240 expected_recv=7890 expected_overflow=0 exact_recv_per_rank=[544, 12325, 11109, 12326, 12330, 2768, 1804, 12330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=10240 expected_recv=10595 expected_overflow=0 exact_recv_per_rank=[544, 12325, 11109, 12326, 12330, 2768, 1804, 12330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=10240 expected_recv=20654 expected_overflow=2090 exact_recv_per_rank=[544, 12325, 11109, 12326, 12330, 2768, 1804, 12330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=10240 expected_recv=21119 actual_valid_recv=19072 observed_loss=2086 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=10240 expected_recv=21118 actual_valid_recv=19200 observed_loss=2085 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=10240 expected_recv=19902 actual_valid_recv=19072 observed_loss=869 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=10240 expected_recv=21121 expected_overflow=2090 exact_recv_per_rank=[544, 12325, 11109, 12326, 12330, 2768, 1804, 12330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=10240 expected_recv=8673 actual_valid_recv=8960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=10240 expected_recv=7890 actual_valid_recv=8064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=19072 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=10240 expected_recv=10595 actual_valid_recv=11008 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=19200 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=19072 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=10240 expected_recv=20654 actual_valid_recv=18688 observed_loss=2090 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=8960 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=8064 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=11008 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=10240 expected_recv=21121 actual_valid_recv=19072 observed_loss=2090 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=18688 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=19072 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8273, 8275, 7949, 8275, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8273, 8275, 7949, 8275, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8273, 8275, 7949, 8275, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8273, 8275, 7949, 8275, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8273, 8275, 7949, 8275, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8273, 8275, 7949, 8275, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=10240 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8273, 8275, 7949, 8275, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=10240 expected_recv=15895 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8273, 8275, 7949, 8275, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=10240 expected_recv=16217 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=10240 expected_recv=15895 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7948, 8275, 8274, 8275, 8275, 7940, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=10240 expected_recv=16221 expected_overflow=0 exact_recv_per_rank=[7948, 8275, 8274, 8275, 8275, 7940, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[7948, 8275, 8274, 8275, 8275, 7940, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7948, 8275, 8274, 8275, 8275, 7940, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=10240 expected_recv=15884 expected_overflow=0 exact_recv_per_rank=[7948, 8275, 8274, 8275, 8275, 7940, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=10240 expected_recv=16221 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=10240 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[7948, 8275, 8274, 8275, 8275, 7940, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7948, 8275, 8274, 8275, 8275, 7940, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=10240 expected_recv=15884 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7948, 8275, 8274, 8275, 8275, 7940, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=10240 expected_recv=16220 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=10240 expected_recv=15903 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7935, 7957, 8273, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7935, 7957, 8273, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7935, 7957, 8273, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=10240 expected_recv=15903 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=10240 expected_recv=16209 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7935, 7957, 8273, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7935, 7957, 8273, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7935, 7957, 8273, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7935, 7957, 8273, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=10240 expected_recv=16209 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7935, 7957, 8273, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8274, 8275, 7944, 8275, 8274, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=10240 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8274, 8275, 7944, 8275, 8274, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=10240 expected_recv=16544 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8274, 8275, 7944, 8275, 8274, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8274, 8275, 7944, 8275, 8274, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8274, 8275, 7944, 8275, 8274, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=10240 expected_recv=15893 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8274, 8275, 7944, 8275, 8274, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=10240 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8274, 8275, 7944, 8275, 8274, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8274, 8275, 7944, 8275, 8274, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=10240 expected_recv=15893 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=10240 expected_recv=16217 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 7948, 8274, 8275, 8274, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=10240 expected_recv=15893 expected_overflow=0 exact_recv_per_rank=[8275, 7948, 8274, 8275, 8274, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=10240 expected_recv=15893 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 7948, 8274, 8275, 8274, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 7948, 8274, 8275, 8274, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=10240 expected_recv=15890 expected_overflow=0 exact_recv_per_rank=[8275, 7948, 8274, 8275, 8274, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 7948, 8274, 8275, 8274, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 7948, 8274, 8275, 8274, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 7948, 8274, 8275, 8274, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=10240 expected_recv=15890 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=10240 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8274, 7950, 8274, 7940, 8275, 8275, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=10240 expected_recv=16216 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=10240 expected_recv=15898 expected_overflow=0 exact_recv_per_rank=[8274, 7950, 8274, 7940, 8275, 8275, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=10240 expected_recv=15898 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 7950, 8274, 7940, 8275, 8275, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 7950, 8274, 7940, 8275, 8275, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 7950, 8274, 7940, 8275, 8275, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 7950, 8274, 7940, 8275, 8275, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8274, 7950, 8274, 7940, 8275, 8275, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 7950, 8274, 7940, 8275, 8275, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7950, 8275, 7940, 8274, 8274, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[7950, 8275, 7940, 8274, 8274, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=10240 expected_recv=15897 expected_overflow=0 exact_recv_per_rank=[7950, 8275, 7940, 8274, 8274, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=10240 expected_recv=15897 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=10240 expected_recv=16215 expected_overflow=0 exact_recv_per_rank=[7950, 8275, 7940, 8274, 8274, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[7950, 8275, 7940, 8274, 8274, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7950, 8275, 7940, 8274, 8274, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7950, 8275, 7940, 8274, 8274, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=10240 expected_recv=16215 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7950, 8275, 7940, 8274, 8274, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=10240 expected_recv=15897 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7951, 8273, 8276, 7940, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7951, 8273, 8276, 7940, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=10240 expected_recv=15897 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7951, 8273, 8276, 7940, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7951, 8273, 8276, 7940, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=10240 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7951, 8273, 8276, 7940, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=10240 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7951, 8273, 8276, 7940, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7951, 8273, 8276, 7940, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=10240 expected_recv=15884 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7951, 8273, 8276, 7940, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=10240 expected_recv=15898 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8275, 7949, 8272, 8275, 7940, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8275, 7949, 8272, 8275, 7940, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=10240 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=10240 expected_recv=16545 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=10240 expected_recv=15884 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=10240 expected_recv=15898 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8275, 7949, 8272, 8275, 7940, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8275, 7949, 8272, 8275, 7940, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8275, 7949, 8272, 8275, 7940, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=10240 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8275, 7949, 8272, 8275, 7940, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8275, 7949, 8272, 8275, 7940, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=10240 expected_recv=15884 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8275, 7949, 8272, 8275, 7940, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=10240 expected_recv=16223 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7950, 8275, 8275, 8273, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7950, 8275, 8275, 8273, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=10240 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=10240 expected_recv=15884 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=10240 expected_recv=16223 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7950, 8275, 8275, 8273, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7950, 8275, 8275, 8273, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=10240 expected_recv=15883 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7950, 8275, 8275, 8273, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7950, 8275, 8275, 8273, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7950, 8275, 8275, 8273, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7950, 8275, 8275, 8273, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 8272, 7951, 8272, 7940, 8275, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=10240 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8276, 8272, 7951, 8272, 7940, 8275, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=10240 expected_recv=15883 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=10240 expected_recv=16545 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8272, 7951, 8272, 7940, 8275, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=10240 expected_recv=15899 expected_overflow=0 exact_recv_per_rank=[8276, 8272, 7951, 8272, 7940, 8275, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=10240 expected_recv=15884 expected_overflow=0 exact_recv_per_rank=[8276, 8272, 7951, 8272, 7940, 8275, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 8272, 7951, 8272, 7940, 8275, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 8272, 7951, 8272, 7940, 8275, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8272, 7951, 8272, 7940, 8275, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=10240 expected_recv=15899 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7941, 8274, 8275, 8274, 8276, 8274, 8272, 7950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7941, 8274, 8275, 8274, 8276, 8274, 8272, 7950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=10240 expected_recv=15884 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=10240 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[7941, 8274, 8275, 8274, 8276, 8274, 8272, 7950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=10240 expected_recv=16217 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7941, 8274, 8275, 8274, 8276, 8274, 8272, 7950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7941, 8274, 8275, 8274, 8276, 8274, 8272, 7950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=10240 expected_recv=15898 expected_overflow=0 exact_recv_per_rank=[7941, 8274, 8275, 8274, 8276, 8274, 8272, 7950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=10240 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[7941, 8274, 8275, 8274, 8276, 8274, 8272, 7950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7941, 8274, 8275, 8274, 8276, 8274, 8272, 7950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=10240 expected_recv=16225 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7952, 8274, 8272, 7940, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7952, 8274, 8272, 7940, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=10240 expected_recv=15898 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7952, 8274, 8272, 7940, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=10240 expected_recv=16216 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=10240 expected_recv=16225 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7952, 8274, 8272, 7940, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7952, 8274, 8272, 7940, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7952, 8274, 8272, 7940, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=10240 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7952, 8274, 8272, 7940, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=10240 expected_recv=15905 expected_overflow=0 exact_recv_per_rank=[8275, 7933, 8273, 7959, 8274, 8273, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=10240 expected_recv=15884 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7952, 8274, 8272, 7940, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=10240 expected_recv=16208 expected_overflow=0 exact_recv_per_rank=[8275, 7933, 8273, 7959, 8274, 8273, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=10240 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 7933, 8273, 7959, 8274, 8273, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=10240 expected_recv=15905 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=10240 expected_recv=15884 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=10240 expected_recv=16208 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 7933, 8273, 7959, 8274, 8273, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 7933, 8273, 7959, 8274, 8273, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 7933, 8273, 7959, 8274, 8273, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 7933, 8273, 7959, 8274, 8273, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8274, 8274, 7952, 7942, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8275, 7933, 8273, 7959, 8274, 8273, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8274, 8274, 7952, 7942, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8274, 8274, 7952, 7942, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=10240 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8274, 8274, 7952, 7942, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8274, 8274, 7952, 7942, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=10240 expected_recv=16545 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=10240 expected_recv=15899 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8274, 8274, 7952, 7942, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=10240 expected_recv=15886 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8274, 8274, 7952, 7942, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=10240 expected_recv=15899 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=10240 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8274, 8274, 7952, 7942, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[544, 31, 11109, 26457, 24624, 2768, 3, 0] avg=8192.0000 max=26457 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.229614 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=10240 expected_recv=15886 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=10240 expected_recv=16545 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[544, 31, 11109, 26457, 24624, 2768, 3, 0] avg=8192.0000 max=26457 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.229614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[544, 31, 11109, 26457, 24624, 2768, 3, 0] avg=8192.0000 max=26457 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.229614 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[544, 12325, 11109, 12326, 12330, 2768, 1804, 12330] avg=8192.0000 max=12330 max_rank=4 min=544 min_rank=0 imbalance_ratio=1.505127 max_load_reduction=14127 max_load_reduction_pct=0.533961 moved_to_replicas=62353 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[544, 31, 11109, 26457, 24624, 2768, 3, 0] avg=8192.0000 max=26457 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.229614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[544, 12325, 11109, 12326, 12330, 2768, 1804, 12330] avg=8192.0000 max=12330 max_rank=4 min=544 min_rank=0 imbalance_ratio=1.505127 max_load_reduction=14127 max_load_reduction_pct=0.533961 moved_to_replicas=62353 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[544, 12325, 11109, 12326, 12330, 2768, 1804, 12330] avg=8192.0000 max=12330 max_rank=4 min=544 min_rank=0 imbalance_ratio=1.505127 max_load_reduction=14127 max_load_reduction_pct=0.533961 moved_to_replicas=62353 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[544, 31, 11109, 26457, 24624, 2768, 3, 0] avg=8192.0000 max=26457 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.229614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[544, 31, 11109, 26457, 24624, 2768, 3, 0] avg=8192.0000 max=26457 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.229614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[544, 31, 11109, 26457, 24624, 2768, 3, 0] avg=8192.0000 max=26457 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.229614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[544, 31, 11109, 26457, 24624, 2768, 3, 0] avg=8192.0000 max=26457 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.229614 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[544, 12325, 11109, 12326, 12330, 2768, 1804, 12330] avg=8192.0000 max=12330 max_rank=4 min=544 min_rank=0 imbalance_ratio=1.505127 max_load_reduction=14127 max_load_reduction_pct=0.533961 moved_to_replicas=62353 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[5, 32743, 32762, 0, 16, 0, 6, 4] avg=8192.0000 max=32762 max_rank=2 min=0 min_rank=3 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[544, 12325, 11109, 12326, 12330, 2768, 1804, 12330] avg=8192.0000 max=12330 max_rank=4 min=544 min_rank=0 imbalance_ratio=1.505127 max_load_reduction=14127 max_load_reduction_pct=0.533961 moved_to_replicas=62353 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[5, 32743, 32762, 0, 16, 0, 6, 4] avg=8192.0000 max=32762 max_rank=2 min=0 min_rank=3 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[544, 12325, 11109, 12326, 12330, 2768, 1804, 12330] avg=8192.0000 max=12330 max_rank=4 min=544 min_rank=0 imbalance_ratio=1.505127 max_load_reduction=14127 max_load_reduction_pct=0.533961 moved_to_replicas=62353 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[5, 32743, 32762, 0, 16, 0, 6, 4] avg=8192.0000 max=32762 max_rank=2 min=0 min_rank=3 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[544, 12325, 11109, 12326, 12330, 2768, 1804, 12330] avg=8192.0000 max=12330 max_rank=4 min=544 min_rank=0 imbalance_ratio=1.505127 max_load_reduction=14127 max_load_reduction_pct=0.533961 moved_to_replicas=62353 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[544, 12325, 11109, 12326, 12330, 2768, 1804, 12330] avg=8192.0000 max=12330 max_rank=4 min=544 min_rank=0 imbalance_ratio=1.505127 max_load_reduction=14127 max_load_reduction_pct=0.533961 moved_to_replicas=62353 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[8273, 8274, 8273, 8275, 7949, 8275, 8274, 7943] avg=8192.0000 max=8275 max_rank=3 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[5, 32743, 32762, 0, 16, 0, 6, 4] avg=8192.0000 max=32762 max_rank=2 min=0 min_rank=3 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[8273, 8274, 8273, 8275, 7949, 8275, 8274, 7943] avg=8192.0000 max=8275 max_rank=3 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[8273, 8274, 8273, 8275, 7949, 8275, 8274, 7943] avg=8192.0000 max=8275 max_rank=3 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[5, 32743, 32762, 0, 16, 0, 6, 4] avg=8192.0000 max=32762 max_rank=2 min=0 min_rank=3 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[5, 32743, 32762, 0, 16, 0, 6, 4] avg=8192.0000 max=32762 max_rank=2 min=0 min_rank=3 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[8273, 8274, 8273, 8275, 7949, 8275, 8274, 7943] avg=8192.0000 max=8275 max_rank=3 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[5, 32743, 32762, 0, 16, 0, 6, 4] avg=8192.0000 max=32762 max_rank=2 min=0 min_rank=3 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[5, 32743, 32762, 0, 16, 0, 6, 4] avg=8192.0000 max=32762 max_rank=2 min=0 min_rank=3 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[4, 0, 32764, 32764, 0, 0, 1, 3] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[8273, 8274, 8273, 8275, 7949, 8275, 8274, 7943] avg=8192.0000 max=8275 max_rank=3 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[4, 0, 32764, 32764, 0, 0, 1, 3] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[8273, 8274, 8273, 8275, 7949, 8275, 8274, 7943] avg=8192.0000 max=8275 max_rank=3 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[4, 0, 32764, 32764, 0, 0, 1, 3] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[8273, 8274, 8273, 8275, 7949, 8275, 8274, 7943] avg=8192.0000 max=8275 max_rank=3 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[8273, 8274, 8273, 8275, 7949, 8275, 8274, 7943] avg=8192.0000 max=8275 max_rank=3 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97936 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[7948, 8275, 8274, 8275, 8275, 7940, 8274, 8275] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[4, 0, 32764, 32764, 0, 0, 1, 3] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[7948, 8275, 8274, 8275, 8275, 7940, 8274, 8275] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[7948, 8275, 8274, 8275, 8275, 7940, 8274, 8275] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[4, 0, 32764, 32764, 0, 0, 1, 3] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[4, 0, 32764, 32764, 0, 0, 1, 3] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[7948, 8275, 8274, 8275, 8275, 7940, 8274, 8275] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[4, 0, 32764, 32764, 0, 0, 1, 3] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[4, 0, 32764, 32764, 0, 0, 1, 3] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[7948, 8275, 8274, 8275, 8275, 7940, 8274, 8275] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[0, 1, 4, 747, 32752, 32016, 12, 4] avg=8192.0000 max=32752 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[7948, 8275, 8274, 8275, 8275, 7940, 8274, 8275] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[0, 1, 4, 747, 32752, 32016, 12, 4] avg=8192.0000 max=32752 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[0, 1, 4, 747, 32752, 32016, 12, 4] avg=8192.0000 max=32752 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[7948, 8275, 8274, 8275, 8275, 7940, 8274, 8275] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[7948, 8275, 8274, 8275, 8275, 7940, 8274, 8275] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[8275, 8274, 7935, 7957, 8273, 8274, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7935 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=96132 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[0, 1, 4, 747, 32752, 32016, 12, 4] avg=8192.0000 max=32752 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[8275, 8274, 7935, 7957, 8273, 8274, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7935 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=96132 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[8275, 8274, 7935, 7957, 8273, 8274, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7935 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=96132 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[0, 1, 4, 747, 32752, 32016, 12, 4] avg=8192.0000 max=32752 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[0, 1, 4, 747, 32752, 32016, 12, 4] avg=8192.0000 max=32752 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[8275, 8274, 7935, 7957, 8273, 8274, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7935 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=96132 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[0, 1, 4, 747, 32752, 32016, 12, 4] avg=8192.0000 max=32752 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[0, 1, 4, 747, 32752, 32016, 12, 4] avg=8192.0000 max=32752 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[3, 1, 32763, 32763, 1, 0, 2, 3] avg=8192.0000 max=32763 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[8275, 8274, 7935, 7957, 8273, 8274, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7935 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=96132 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[8275, 8274, 7935, 7957, 8273, 8274, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7935 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=96132 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[3, 1, 32763, 32763, 1, 0, 2, 3] avg=8192.0000 max=32763 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[3, 1, 32763, 32763, 1, 0, 2, 3] avg=8192.0000 max=32763 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[8275, 8274, 7935, 7957, 8273, 8274, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7935 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=96132 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[3, 1, 32763, 32763, 1, 0, 2, 3] avg=8192.0000 max=32763 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[8275, 8274, 7935, 7957, 8273, 8274, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7935 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=96132 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[8275, 8272, 8274, 8275, 7944, 8275, 8274, 7947] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[8275, 8272, 8274, 8275, 7944, 8275, 8274, 7947] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[8275, 8272, 8274, 8275, 7944, 8275, 8274, 7947] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[3, 1, 32763, 32763, 1, 0, 2, 3] avg=8192.0000 max=32763 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[3, 1, 32763, 32763, 1, 0, 2, 3] avg=8192.0000 max=32763 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[8275, 8272, 8274, 8275, 7944, 8275, 8274, 7947] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[3, 1, 32763, 32763, 1, 0, 2, 3] avg=8192.0000 max=32763 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[3, 1, 32763, 32763, 1, 0, 2, 3] avg=8192.0000 max=32763 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[0, 4, 4, 37, 30684, 32763, 2, 2042] avg=8192.0000 max=32763 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[8275, 8272, 8274, 8275, 7944, 8275, 8274, 7947] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[0, 4, 4, 37, 30684, 32763, 2, 2042] avg=8192.0000 max=32763 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[0, 4, 4, 37, 30684, 32763, 2, 2042] avg=8192.0000 max=32763 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[8275, 8272, 8274, 8275, 7944, 8275, 8274, 7947] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[8275, 8272, 8274, 8275, 7944, 8275, 8274, 7947] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[0, 4, 4, 37, 30684, 32763, 2, 2042] avg=8192.0000 max=32763 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[8275, 7948, 8274, 8275, 8274, 8272, 8274, 7944] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=92933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[8275, 8272, 8274, 8275, 7944, 8275, 8274, 7947] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[8275, 7948, 8274, 8275, 8274, 8272, 8274, 7944] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=92933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[8275, 7948, 8274, 8275, 8274, 8272, 8274, 7944] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=92933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[0, 4, 4, 37, 30684, 32763, 2, 2042] avg=8192.0000 max=32763 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[8275, 7948, 8274, 8275, 8274, 8272, 8274, 7944] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=92933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[0, 4, 4, 37, 30684, 32763, 2, 2042] avg=8192.0000 max=32763 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[0, 4, 4, 37, 30684, 32763, 2, 2042] avg=8192.0000 max=32763 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[30642, 2120, 0, 1, 32763, 0, 5, 5] avg=8192.0000 max=32763 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[0, 4, 4, 37, 30684, 32763, 2, 2042] avg=8192.0000 max=32763 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[8275, 7948, 8274, 8275, 8274, 8272, 8274, 7944] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=92933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[30642, 2120, 0, 1, 32763, 0, 5, 5] avg=8192.0000 max=32763 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[8275, 7948, 8274, 8275, 8274, 8272, 8274, 7944] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=92933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[30642, 2120, 0, 1, 32763, 0, 5, 5] avg=8192.0000 max=32763 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[8275, 7948, 8274, 8275, 8274, 8272, 8274, 7944] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=92933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[30642, 2120, 0, 1, 32763, 0, 5, 5] avg=8192.0000 max=32763 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[8274, 7950, 8274, 7940, 8275, 8275, 8273, 8275] avg=8192.0000 max=8275 max_rank=4 min=7940 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=93296 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[8275, 7948, 8274, 8275, 8274, 8272, 8274, 7944] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=92933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[8274, 7950, 8274, 7940, 8275, 8275, 8273, 8275] avg=8192.0000 max=8275 max_rank=4 min=7940 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=93296 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[8274, 7950, 8274, 7940, 8275, 8275, 8273, 8275] avg=8192.0000 max=8275 max_rank=4 min=7940 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=93296 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[30642, 2120, 0, 1, 32763, 0, 5, 5] avg=8192.0000 max=32763 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[30642, 2120, 0, 1, 32763, 0, 5, 5] avg=8192.0000 max=32763 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[8274, 7950, 8274, 7940, 8275, 8275, 8273, 8275] avg=8192.0000 max=8275 max_rank=4 min=7940 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=93296 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[30642, 2120, 0, 1, 32763, 0, 5, 5] avg=8192.0000 max=32763 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[8274, 7950, 8274, 7940, 8275, 8275, 8273, 8275] avg=8192.0000 max=8275 max_rank=4 min=7940 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=93296 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[54, 0, 1, 32709, 5, 32763, 4, 0] avg=8192.0000 max=32763 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[30642, 2120, 0, 1, 32763, 0, 5, 5] avg=8192.0000 max=32763 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[54, 0, 1, 32709, 5, 32763, 4, 0] avg=8192.0000 max=32763 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[8274, 7950, 8274, 7940, 8275, 8275, 8273, 8275] avg=8192.0000 max=8275 max_rank=4 min=7940 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=93296 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[54, 0, 1, 32709, 5, 32763, 4, 0] avg=8192.0000 max=32763 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[8274, 7950, 8274, 7940, 8275, 8275, 8273, 8275] avg=8192.0000 max=8275 max_rank=4 min=7940 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=93296 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[54, 0, 1, 32709, 5, 32763, 4, 0] avg=8192.0000 max=32763 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[7950, 8275, 7940, 8274, 8274, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[8274, 7950, 8274, 7940, 8275, 8275, 8273, 8275] avg=8192.0000 max=8275 max_rank=4 min=7940 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=93296 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[7950, 8275, 7940, 8274, 8274, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[54, 0, 1, 32709, 5, 32763, 4, 0] avg=8192.0000 max=32763 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[7950, 8275, 7940, 8274, 8274, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[7950, 8275, 7940, 8274, 8274, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[54, 0, 1, 32709, 5, 32763, 4, 0] avg=8192.0000 max=32763 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[54, 0, 1, 32709, 5, 32763, 4, 0] avg=8192.0000 max=32763 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[7950, 8275, 7940, 8274, 8274, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[5, 32757, 0, 6, 5, 0, 0, 32763] avg=8192.0000 max=32763 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[54, 0, 1, 32709, 5, 32763, 4, 0] avg=8192.0000 max=32763 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[5, 32757, 0, 6, 5, 0, 0, 32763] avg=8192.0000 max=32763 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[7950, 8275, 7940, 8274, 8274, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[5, 32757, 0, 6, 5, 0, 0, 32763] avg=8192.0000 max=32763 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[7950, 8275, 7940, 8274, 8274, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[5, 32757, 0, 6, 5, 0, 0, 32763] avg=8192.0000 max=32763 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[8275, 8274, 8275, 7951, 8273, 8276, 7940, 8272] avg=8192.0000 max=8276 max_rank=5 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[7950, 8275, 7940, 8274, 8274, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[8275, 8274, 8275, 7951, 8273, 8276, 7940, 8272] avg=8192.0000 max=8276 max_rank=5 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[5, 32757, 0, 6, 5, 0, 0, 32763] avg=8192.0000 max=32763 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[8275, 8274, 8275, 7951, 8273, 8276, 7940, 8272] avg=8192.0000 max=8276 max_rank=5 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[8275, 8274, 8275, 7951, 8273, 8276, 7940, 8272] avg=8192.0000 max=8276 max_rank=5 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[5, 32757, 0, 6, 5, 0, 0, 32763] avg=8192.0000 max=32763 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[5, 32757, 0, 6, 5, 0, 0, 32763] avg=8192.0000 max=32763 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[8275, 8274, 8275, 7951, 8273, 8276, 7940, 8272] avg=8192.0000 max=8276 max_rank=5 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[32763, 0, 5, 5, 0, 32763, 0, 0] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[5, 32757, 0, 6, 5, 0, 0, 32763] avg=8192.0000 max=32763 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[32763, 0, 5, 5, 0, 32763, 0, 0] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[8275, 8274, 8275, 7951, 8273, 8276, 7940, 8272] avg=8192.0000 max=8276 max_rank=5 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[32763, 0, 5, 5, 0, 32763, 0, 0] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[8275, 8274, 8275, 7951, 8273, 8276, 7940, 8272] avg=8192.0000 max=8276 max_rank=5 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[32763, 0, 5, 5, 0, 32763, 0, 0] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[8276, 8275, 8275, 7949, 8272, 8275, 7940, 8274] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[8275, 8274, 8275, 7951, 8273, 8276, 7940, 8272] avg=8192.0000 max=8276 max_rank=5 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[32763, 0, 5, 5, 0, 32763, 0, 0] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[8276, 8275, 8275, 7949, 8272, 8275, 7940, 8274] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[8276, 8275, 8275, 7949, 8272, 8275, 7940, 8274] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[8276, 8275, 8275, 7949, 8272, 8275, 7940, 8274] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[8276, 8275, 8275, 7949, 8272, 8275, 7940, 8274] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[32763, 0, 5, 5, 0, 32763, 0, 0] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[5, 32763, 0, 5, 32762, 0, 1, 0] avg=8192.0000 max=32763 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[32763, 0, 5, 5, 0, 32763, 0, 0] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[32763, 0, 5, 5, 0, 32763, 0, 0] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[5, 32763, 0, 5, 32762, 0, 1, 0] avg=8192.0000 max=32763 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[5, 32763, 0, 5, 32762, 0, 1, 0] avg=8192.0000 max=32763 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[8275, 8275, 8274, 7950, 8275, 8275, 8273, 7939] avg=8192.0000 max=8275 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[5, 32763, 0, 5, 32762, 0, 1, 0] avg=8192.0000 max=32763 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[8276, 8275, 8275, 7949, 8272, 8275, 7940, 8274] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[8276, 8275, 8275, 7949, 8272, 8275, 7940, 8274] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[5, 32763, 0, 5, 32762, 0, 1, 0] avg=8192.0000 max=32763 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[8276, 8275, 8275, 7949, 8272, 8275, 7940, 8274] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[8275, 8275, 8274, 7950, 8275, 8275, 8273, 7939] avg=8192.0000 max=8275 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[8275, 8275, 8274, 7950, 8275, 8275, 8273, 7939] avg=8192.0000 max=8275 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[8275, 8275, 8274, 7950, 8275, 8275, 8273, 7939] avg=8192.0000 max=8275 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[8275, 8275, 8274, 7950, 8275, 8275, 8273, 7939] avg=8192.0000 max=8275 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[0, 5, 5, 0, 0, 0, 32763, 32763] avg=8192.0000 max=32763 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[5, 32763, 0, 5, 32762, 0, 1, 0] avg=8192.0000 max=32763 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[5, 32763, 0, 5, 32762, 0, 1, 0] avg=8192.0000 max=32763 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[5, 32763, 0, 5, 32762, 0, 1, 0] avg=8192.0000 max=32763 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[0, 5, 5, 0, 0, 0, 32763, 32763] avg=8192.0000 max=32763 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[0, 5, 5, 0, 0, 0, 32763, 32763] avg=8192.0000 max=32763 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[8276, 8272, 7951, 8272, 7940, 8275, 8275, 8275] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[8275, 8275, 8274, 7950, 8275, 8275, 8273, 7939] avg=8192.0000 max=8275 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[0, 5, 5, 0, 0, 0, 32763, 32763] avg=8192.0000 max=32763 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[0, 5, 5, 0, 0, 0, 32763, 32763] avg=8192.0000 max=32763 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[8275, 8275, 8274, 7950, 8275, 8275, 8273, 7939] avg=8192.0000 max=8275 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[8275, 8275, 8274, 7950, 8275, 8275, 8273, 7939] avg=8192.0000 max=8275 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[8276, 8272, 7951, 8272, 7940, 8275, 8275, 8275] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[8276, 8272, 7951, 8272, 7940, 8275, 8275, 8275] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[8276, 8272, 7951, 8272, 7940, 8275, 8275, 8275] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[8276, 8272, 7951, 8272, 7940, 8275, 8275, 8275] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[1, 3, 32763, 5, 0, 32759, 0, 5] avg=8192.0000 max=32763 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[0, 5, 5, 0, 0, 0, 32763, 32763] avg=8192.0000 max=32763 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[0, 5, 5, 0, 0, 0, 32763, 32763] avg=8192.0000 max=32763 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[0, 5, 5, 0, 0, 0, 32763, 32763] avg=8192.0000 max=32763 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[1, 3, 32763, 5, 0, 32759, 0, 5] avg=8192.0000 max=32763 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[1, 3, 32763, 5, 0, 32759, 0, 5] avg=8192.0000 max=32763 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[7941, 8274, 8275, 8274, 8276, 8274, 8272, 7950] avg=8192.0000 max=8276 max_rank=4 min=7941 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[1, 3, 32763, 5, 0, 32759, 0, 5] avg=8192.0000 max=32763 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[8276, 8272, 7951, 8272, 7940, 8275, 8275, 8275] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[1, 3, 32763, 5, 0, 32759, 0, 5] avg=8192.0000 max=32763 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[8276, 8272, 7951, 8272, 7940, 8275, 8275, 8275] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[8276, 8272, 7951, 8272, 7940, 8275, 8275, 8275] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[7941, 8274, 8275, 8274, 8276, 8274, 8272, 7950] avg=8192.0000 max=8276 max_rank=4 min=7941 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[7941, 8274, 8275, 8274, 8276, 8274, 8272, 7950] avg=8192.0000 max=8276 max_rank=4 min=7941 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[7941, 8274, 8275, 8274, 8276, 8274, 8272, 7950] avg=8192.0000 max=8276 max_rank=4 min=7941 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[7941, 8274, 8275, 8274, 8276, 8274, 8272, 7950] avg=8192.0000 max=8276 max_rank=4 min=7941 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[32763, 0, 5, 6, 32758, 0, 0, 4] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[1, 3, 32763, 5, 0, 32759, 0, 5] avg=8192.0000 max=32763 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[1, 3, 32763, 5, 0, 32759, 0, 5] avg=8192.0000 max=32763 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[1, 3, 32763, 5, 0, 32759, 0, 5] avg=8192.0000 max=32763 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[32763, 0, 5, 6, 32758, 0, 0, 4] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[32763, 0, 5, 6, 32758, 0, 0, 4] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[8276, 8275, 8273, 7952, 8274, 8272, 7940, 8274] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97958 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[32763, 0, 5, 6, 32758, 0, 0, 4] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[7941, 8274, 8275, 8274, 8276, 8274, 8272, 7950] avg=8192.0000 max=8276 max_rank=4 min=7941 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[32763, 0, 5, 6, 32758, 0, 0, 4] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[7941, 8274, 8275, 8274, 8276, 8274, 8272, 7950] avg=8192.0000 max=8276 max_rank=4 min=7941 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[7941, 8274, 8275, 8274, 8276, 8274, 8272, 7950] avg=8192.0000 max=8276 max_rank=4 min=7941 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[8276, 8275, 8273, 7952, 8274, 8272, 7940, 8274] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97958 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[8276, 8275, 8273, 7952, 8274, 8272, 7940, 8274] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97958 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[8276, 8275, 8273, 7952, 8274, 8272, 7940, 8274] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97958 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[8276, 8275, 8273, 7952, 8274, 8272, 7940, 8274] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97958 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[0, 1, 32728, 39, 7, 5, 0, 32756] avg=8192.0000 max=32756 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[32763, 0, 5, 6, 32758, 0, 0, 4] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[32763, 0, 5, 6, 32758, 0, 0, 4] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[32763, 0, 5, 6, 32758, 0, 0, 4] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[0, 1, 32728, 39, 7, 5, 0, 32756] avg=8192.0000 max=32756 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[0, 1, 32728, 39, 7, 5, 0, 32756] avg=8192.0000 max=32756 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[8275, 7933, 8273, 7959, 8274, 8273, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7933 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[0, 1, 32728, 39, 7, 5, 0, 32756] avg=8192.0000 max=32756 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[8276, 8275, 8273, 7952, 8274, 8272, 7940, 8274] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97958 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[8276, 8275, 8273, 7952, 8274, 8272, 7940, 8274] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97958 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[0, 1, 32728, 39, 7, 5, 0, 32756] avg=8192.0000 max=32756 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[8276, 8275, 8273, 7952, 8274, 8272, 7940, 8274] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97958 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[8275, 7933, 8273, 7959, 8274, 8273, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7933 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[8275, 7933, 8273, 7959, 8274, 8273, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7933 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[8275, 7933, 8273, 7959, 8274, 8273, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7933 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[5, 32756, 0, 0, 9, 3, 3, 32760] avg=8192.0000 max=32760 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[8275, 7933, 8273, 7959, 8274, 8273, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7933 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[0, 1, 32728, 39, 7, 5, 0, 32756] avg=8192.0000 max=32756 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[0, 1, 32728, 39, 7, 5, 0, 32756] avg=8192.0000 max=32756 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[5, 32756, 0, 0, 9, 3, 3, 32760] avg=8192.0000 max=32760 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[0, 1, 32728, 39, 7, 5, 0, 32756] avg=8192.0000 max=32756 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[5, 32756, 0, 0, 9, 3, 3, 32760] avg=8192.0000 max=32760 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[5, 32756, 0, 0, 9, 3, 3, 32760] avg=8192.0000 max=32760 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[8273, 8275, 8274, 8274, 7952, 7942, 8273, 8273] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24485 max_load_reduction_pct=0.747405 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[8275, 7933, 8273, 7959, 8274, 8273, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7933 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[8275, 7933, 8273, 7959, 8274, 8273, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7933 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[8273, 8275, 8274, 8274, 7952, 7942, 8273, 8273] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24485 max_load_reduction_pct=0.747405 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[8275, 7933, 8273, 7959, 8274, 8273, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7933 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[5, 32756, 0, 0, 9, 3, 3, 32760] avg=8192.0000 max=32760 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[8273, 8275, 8274, 8274, 7952, 7942, 8273, 8273] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24485 max_load_reduction_pct=0.747405 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[8273, 8275, 8274, 8274, 7952, 7942, 8273, 8273] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24485 max_load_reduction_pct=0.747405 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[5, 32756, 0, 0, 9, 3, 3, 32760] avg=8192.0000 max=32760 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[8273, 8275, 8274, 8274, 7952, 7942, 8273, 8273] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24485 max_load_reduction_pct=0.747405 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[5, 32756, 0, 0, 9, 3, 3, 32760] avg=8192.0000 max=32760 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[5, 32756, 0, 0, 9, 3, 3, 32760] avg=8192.0000 max=32760 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[8273, 8275, 8274, 8274, 7952, 7942, 8273, 8273] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24485 max_load_reduction_pct=0.747405 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[8273, 8275, 8274, 8274, 7952, 7942, 8273, 8273] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24485 max_load_reduction_pct=0.747405 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[8273, 8275, 8274, 8274, 7952, 7942, 8273, 8273] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24485 max_load_reduction_pct=0.747405 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=32 total_expected_recv=265773 total_dropped_tokens=2086 drop_rate_pct=0.784880 records_with_drop_pct=3.125000 peak_rank_overflow=2086 total_slack_slots=60032 peak_demand=12326 required_capacity=1.504639 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=32 total_expected_recv=267041 total_dropped_tokens=2090 drop_rate_pct=0.782651 records_with_drop_pct=3.125000 peak_rank_overflow=2090 total_slack_slots=59520 peak_demand=12330 required_capacity=1.505127 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=32 total_expected_recv=264918 total_dropped_tokens=2090 drop_rate_pct=0.788923 records_with_drop_pct=3.125000 peak_rank_overflow=2090 total_slack_slots=60928 peak_demand=12330 required_capacity=1.505127 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=32 total_expected_recv=267703 total_dropped_tokens=2085 drop_rate_pct=0.778848 records_with_drop_pct=3.125000 peak_rank_overflow=2085 total_slack_slots=59136 peak_demand=12325 required_capacity=1.504517 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=32 total_expected_recv=255610 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=68480 peak_demand=8276 required_capacity=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=32 total_expected_recv=255496 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=67712 peak_demand=8791 required_capacity=1.073120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=32 total_expected_recv=266814 total_dropped_tokens=869 drop_rate_pct=0.325695 records_with_drop_pct=3.125000 peak_rank_overflow=869 total_slack_slots=58752 peak_demand=11109 required_capacity=1.356079 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=32 total_expected_recv=253797 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=71424 peak_demand=8276 required_capacity=1.010254 + [2026-08-30 05:08:04] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 888.5 | throughput per GPU (TFLOP/s/GPU): 375.2 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.637150E+00 | loss scale: 1.0 | grad norm: 6.697 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 10240 | min safe capacity: 1.2500 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=10240 expected_recv=7797 expected_overflow=0 exact_recv_per_rank=[615, 2070, 12008, 12007, 12009, 3902, 12007, 10918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=10240 expected_recv=23560 expected_overflow=3080 exact_recv_per_rank=[615, 2070, 12008, 12007, 12009, 3902, 12007, 10918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=10240 expected_recv=23559 expected_overflow=3079 exact_recv_per_rank=[615, 2070, 12008, 12007, 12009, 3902, 12007, 10918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=10240 expected_recv=7152 expected_overflow=0 exact_recv_per_rank=[615, 2070, 12008, 12007, 12009, 3902, 12007, 10918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=10240 expected_recv=20539 expected_overflow=678 exact_recv_per_rank=[615, 2070, 12008, 12007, 12009, 3902, 12007, 10918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=10240 expected_recv=23559 expected_overflow=3079 exact_recv_per_rank=[615, 2070, 12008, 12007, 12009, 3902, 12007, 10918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=10240 expected_recv=1345 expected_overflow=0 exact_recv_per_rank=[615, 2070, 12008, 12007, 12009, 3902, 12007, 10918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=10240 expected_recv=7797 actual_valid_recv=8192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=10240 expected_recv=23561 expected_overflow=3081 exact_recv_per_rank=[615, 2070, 12008, 12007, 12009, 3902, 12007, 10918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=10240 expected_recv=23560 actual_valid_recv=20480 observed_loss=3080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=10240 expected_recv=23559 actual_valid_recv=20480 observed_loss=3079 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=10240 expected_recv=7152 actual_valid_recv=7296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=10240 expected_recv=20539 actual_valid_recv=20096 observed_loss=678 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=8192 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=10240 expected_recv=23559 actual_valid_recv=20480 observed_loss=3079 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=20480 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=20480 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=7296 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=10240 expected_recv=1345 actual_valid_recv=1408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=20096 slack_slots=384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=10240 expected_recv=23561 actual_valid_recv=20480 observed_loss=3081 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=20480 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=1408 slack_slots=19072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=20480 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8272, 8275, 7946, 7944, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8272, 8275, 7946, 7944, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8272, 8275, 7946, 7944, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=10240 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8272, 8275, 7946, 7944, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8272, 8275, 7946, 7944, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=10240 expected_recv=16216 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8272, 8275, 7946, 7944, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8272, 8275, 7946, 7944, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=10240 expected_recv=15894 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8272, 8275, 7946, 7944, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=10240 expected_recv=15894 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8275, 8272, 7949, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8275, 8272, 7949, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8275, 8272, 7949, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=10240 expected_recv=16225 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8275, 8272, 7949, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=10240 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8275, 8272, 7949, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=10240 expected_recv=16225 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=10240 expected_recv=16216 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=10240 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8275, 8272, 7949, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8275, 8272, 7949, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=10240 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8275, 8272, 7949, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=10240 expected_recv=16216 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=10240 expected_recv=16216 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=10240 expected_recv=16871 expected_overflow=0 exact_recv_per_rank=[6396, 8596, 8596, 8597, 8596, 7564, 8595, 8596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=10240 expected_recv=16872 expected_overflow=0 exact_recv_per_rank=[6396, 8596, 8596, 8597, 8596, 7564, 8595, 8596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=10240 expected_recv=16871 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=10240 expected_recv=15760 expected_overflow=0 exact_recv_per_rank=[6396, 8596, 8596, 8597, 8596, 7564, 8595, 8596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=10240 expected_recv=16292 expected_overflow=0 exact_recv_per_rank=[6396, 8596, 8596, 8597, 8596, 7564, 8595, 8596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=10240 expected_recv=16870 expected_overflow=0 exact_recv_per_rank=[6396, 8596, 8596, 8597, 8596, 7564, 8595, 8596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=10240 expected_recv=16872 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=10240 expected_recv=15760 actual_valid_recv=16000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=10240 expected_recv=16292 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=10240 expected_recv=16870 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=10240 expected_recv=16868 expected_overflow=0 exact_recv_per_rank=[6396, 8596, 8596, 8597, 8596, 7564, 8595, 8596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16000 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=10240 expected_recv=14671 expected_overflow=0 exact_recv_per_rank=[6396, 8596, 8596, 8597, 8596, 7564, 8595, 8596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=10240 expected_recv=16868 expected_overflow=0 exact_recv_per_rank=[6396, 8596, 8596, 8597, 8596, 7564, 8595, 8596] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17152 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=10240 expected_recv=16868 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=10240 expected_recv=14671 actual_valid_recv=14848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=10240 expected_recv=16868 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17280 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=14848 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17024 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8275, 8272, 7944, 7949, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8275, 8272, 7944, 7949, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=10240 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8275, 8272, 7944, 7949, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=10240 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8275, 8272, 7944, 7949, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8275, 8272, 7944, 7949, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=10240 expected_recv=16221 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8275, 8272, 7944, 7949, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=10240 expected_recv=16220 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8275, 8272, 7944, 7949, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=10240 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8275, 8272, 7944, 7949, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=10240 expected_recv=16216 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=10240 expected_recv=16221 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=10240 expected_recv=16216 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7947, 7944, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=10240 expected_recv=15894 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7947, 7944, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=10240 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7947, 7944, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=10240 expected_recv=15894 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7947, 7944, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7947, 7944, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=10240 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7947, 7944, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=10240 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=10240 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7947, 7944, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7947, 7944, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=10240 expected_recv=16217 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=10240 expected_recv=16217 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7945, 8275, 8276, 8275, 8275, 8272, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[7945, 8275, 8276, 8275, 8275, 8272, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[7945, 8275, 8276, 8275, 8275, 8272, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=10240 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[7945, 8275, 8276, 8275, 8275, 8272, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=10240 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[7945, 8275, 8276, 8275, 8275, 8272, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[7945, 8275, 8276, 8275, 8275, 8272, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7945, 8275, 8276, 8275, 8275, 8272, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=10240 expected_recv=16216 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=10240 expected_recv=16545 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=10240 expected_recv=15892 expected_overflow=0 exact_recv_per_rank=[7945, 8275, 8276, 8275, 8275, 8272, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=10240 expected_recv=15892 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7943, 8273, 8275, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=10240 expected_recv=15887 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7943, 8273, 8275, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=10240 expected_recv=15887 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7943, 8273, 8275, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7943, 8273, 8275, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7943, 8273, 8275, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=10240 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7943, 8273, 8275, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7943, 8273, 8275, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7943, 8273, 8275, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=10240 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=10240 expected_recv=15893 expected_overflow=0 exact_recv_per_rank=[8274, 7947, 8276, 8275, 8273, 8275, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 7947, 8276, 8275, 8273, 8275, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=10240 expected_recv=15893 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=10240 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8274, 7947, 8276, 8275, 8273, 8275, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 7947, 8276, 8275, 8273, 8275, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=10240 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8274, 7947, 8276, 8275, 8273, 8275, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=10240 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8274, 7947, 8276, 8275, 8273, 8275, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8274, 7947, 8276, 8275, 8273, 8275, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 7947, 8276, 8275, 8273, 8275, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=10240 expected_recv=16217 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[7947, 8276, 8273, 8274, 8275, 8275, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[7947, 8276, 8273, 8274, 8275, 8275, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[7947, 8276, 8273, 8274, 8275, 8275, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7947, 8276, 8273, 8274, 8275, 8275, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[7947, 8276, 8273, 8274, 8275, 8275, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7947, 8276, 8273, 8274, 8275, 8275, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=10240 expected_recv=15893 expected_overflow=0 exact_recv_per_rank=[7947, 8276, 8273, 8274, 8275, 8275, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7947, 8276, 8273, 8274, 8275, 8275, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 7946, 8274, 8274, 8275, 8274, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=10240 expected_recv=15893 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 7946, 8274, 8274, 8275, 8274, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=10240 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 7946, 8274, 8274, 8275, 8274, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 7946, 8274, 8274, 8275, 8274, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=10240 expected_recv=16545 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 7946, 8274, 8274, 8275, 8274, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 7946, 8274, 8274, 8275, 8274, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=10240 expected_recv=16221 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 7946, 8274, 8274, 8275, 8274, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=10240 expected_recv=15891 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 7946, 8274, 8274, 8275, 8274, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8273, 7946, 8275, 7944, 8275, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8273, 7946, 8275, 7944, 8275, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=10240 expected_recv=16221 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=10240 expected_recv=15891 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 8273, 7946, 8275, 7944, 8275, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8276, 8273, 7946, 8275, 7944, 8275, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=10240 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8276, 8273, 7946, 8275, 7944, 8275, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=10240 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8273, 7946, 8275, 7944, 8275, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8273, 7946, 8275, 7944, 8275, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=10240 expected_recv=15890 expected_overflow=0 exact_recv_per_rank=[8276, 8273, 7946, 8275, 7944, 8275, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=10240 expected_recv=16220 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=10240 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=10240 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8276, 7944, 8274, 7945, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8276, 7944, 8274, 7945, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=10240 expected_recv=15890 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=10240 expected_recv=15888 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8276, 7944, 8274, 7945, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=10240 expected_recv=15891 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8276, 7944, 8274, 7945, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=10240 expected_recv=15891 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8276, 7944, 8274, 7945, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8276, 7944, 8274, 7945, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8276, 7944, 8274, 7945, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8276, 7944, 8274, 7945, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 7945, 8274, 7946, 8272, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 7945, 8274, 7946, 8272, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 7945, 8274, 7946, 8272, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=10240 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 7945, 8274, 7946, 8272, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=10240 expected_recv=16545 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=10240 expected_recv=15892 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 7945, 8274, 7946, 8272, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 7945, 8274, 7946, 8272, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=10240 expected_recv=15892 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 7945, 8274, 7946, 8272, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=10240 expected_recv=15931 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7986, 8274, 8274, 7906, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=10240 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 7945, 8274, 7946, 8272, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7986, 8274, 8274, 7906, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7986, 8274, 8274, 7906, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=10240 expected_recv=15931 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7986, 8274, 8274, 7906, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=10240 expected_recv=16217 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7986, 8274, 8274, 7906, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=10240 expected_recv=16180 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7986, 8274, 8274, 7906, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=10240 expected_recv=16180 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7986, 8274, 8274, 7906, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8273, 7947, 8276, 8275, 8272, 7944, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7986, 8274, 8274, 7906, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=10240 expected_recv=15892 expected_overflow=0 exact_recv_per_rank=[8273, 7947, 8276, 8275, 8272, 7944, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 7947, 8276, 8275, 8272, 7944, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8273, 7947, 8276, 8275, 8272, 7944, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=10240 expected_recv=15892 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=10240 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8273, 7947, 8276, 8275, 8272, 7944, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 7947, 8276, 8275, 8272, 7944, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=10240 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 7947, 8276, 8275, 8272, 7944, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[615, 8, 14070, 22922, 24015, 3902, 1, 3] avg=8192.0000 max=24015 max_rank=4 min=1 min_rank=6 imbalance_ratio=2.931519 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8273, 7947, 8276, 8275, 8272, 7944, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[615, 8, 14070, 22922, 24015, 3902, 1, 3] avg=8192.0000 max=24015 max_rank=4 min=1 min_rank=6 imbalance_ratio=2.931519 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[615, 8, 14070, 22922, 24015, 3902, 1, 3] avg=8192.0000 max=24015 max_rank=4 min=1 min_rank=6 imbalance_ratio=2.931519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[615, 8, 14070, 22922, 24015, 3902, 1, 3] avg=8192.0000 max=24015 max_rank=4 min=1 min_rank=6 imbalance_ratio=2.931519 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[615, 8, 14070, 22922, 24015, 3902, 1, 3] avg=8192.0000 max=24015 max_rank=4 min=1 min_rank=6 imbalance_ratio=2.931519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[615, 2070, 12008, 12007, 12009, 3902, 12007, 10918] avg=8192.0000 max=12009 max_rank=4 min=615 min_rank=0 imbalance_ratio=1.465942 max_load_reduction=12006 max_load_reduction_pct=0.499938 moved_to_replicas=51879 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[615, 8, 14070, 22922, 24015, 3902, 1, 3] avg=8192.0000 max=24015 max_rank=4 min=1 min_rank=6 imbalance_ratio=2.931519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[615, 2070, 12008, 12007, 12009, 3902, 12007, 10918] avg=8192.0000 max=12009 max_rank=4 min=615 min_rank=0 imbalance_ratio=1.465942 max_load_reduction=12006 max_load_reduction_pct=0.499938 moved_to_replicas=51879 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[615, 2070, 12008, 12007, 12009, 3902, 12007, 10918] avg=8192.0000 max=12009 max_rank=4 min=615 min_rank=0 imbalance_ratio=1.465942 max_load_reduction=12006 max_load_reduction_pct=0.499938 moved_to_replicas=51879 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[615, 2070, 12008, 12007, 12009, 3902, 12007, 10918] avg=8192.0000 max=12009 max_rank=4 min=615 min_rank=0 imbalance_ratio=1.465942 max_load_reduction=12006 max_load_reduction_pct=0.499938 moved_to_replicas=51879 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[615, 8, 14070, 22922, 24015, 3902, 1, 3] avg=8192.0000 max=24015 max_rank=4 min=1 min_rank=6 imbalance_ratio=2.931519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[615, 2070, 12008, 12007, 12009, 3902, 12007, 10918] avg=8192.0000 max=12009 max_rank=4 min=615 min_rank=0 imbalance_ratio=1.465942 max_load_reduction=12006 max_load_reduction_pct=0.499938 moved_to_replicas=51879 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[615, 2070, 12008, 12007, 12009, 3902, 12007, 10918] avg=8192.0000 max=12009 max_rank=4 min=615 min_rank=0 imbalance_ratio=1.465942 max_load_reduction=12006 max_load_reduction_pct=0.499938 moved_to_replicas=51879 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[615, 8, 14070, 22922, 24015, 3902, 1, 3] avg=8192.0000 max=24015 max_rank=4 min=1 min_rank=6 imbalance_ratio=2.931519 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[0, 32757, 32767, 0, 10, 0, 2, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[0, 32757, 32767, 0, 10, 0, 2, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[0, 32757, 32767, 0, 10, 0, 2, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[615, 2070, 12008, 12007, 12009, 3902, 12007, 10918] avg=8192.0000 max=12009 max_rank=4 min=615 min_rank=0 imbalance_ratio=1.465942 max_load_reduction=12006 max_load_reduction_pct=0.499938 moved_to_replicas=51879 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[0, 32757, 32767, 0, 10, 0, 2, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[615, 2070, 12008, 12007, 12009, 3902, 12007, 10918] avg=8192.0000 max=12009 max_rank=4 min=615 min_rank=0 imbalance_ratio=1.465942 max_load_reduction=12006 max_load_reduction_pct=0.499938 moved_to_replicas=51879 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[0, 32757, 32767, 0, 10, 0, 2, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[8276, 8274, 8272, 8275, 7946, 7944, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[8276, 8274, 8272, 8275, 7946, 7944, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[0, 32757, 32767, 0, 10, 0, 2, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[8276, 8274, 8272, 8275, 7946, 7944, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[8276, 8274, 8272, 8275, 7946, 7944, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[8276, 8274, 8272, 8275, 7946, 7944, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[0, 32757, 32767, 0, 10, 0, 2, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[8276, 8274, 8272, 8275, 7946, 7944, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[0, 32757, 32767, 0, 10, 0, 2, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 1, 0, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 1, 0, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 1, 0, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 1, 0, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[8276, 8274, 8272, 8275, 7946, 7944, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[8276, 8274, 8272, 8275, 7946, 7944, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 1, 0, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[8273, 8276, 8275, 8275, 8272, 7949, 7944, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[8273, 8276, 8275, 8275, 8272, 7949, 7944, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[8273, 8276, 8275, 8275, 8272, 7949, 7944, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 1, 0, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[8273, 8276, 8275, 8275, 8272, 7949, 7944, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[8273, 8276, 8275, 8275, 8272, 7949, 7944, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[8273, 8276, 8275, 8275, 8272, 7949, 7944, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 1, 0, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 1, 0, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[1, 0, 0, 27413, 32183, 5932, 6, 1] avg=8192.0000 max=32183 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.928589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[1, 0, 0, 27413, 32183, 5932, 6, 1] avg=8192.0000 max=32183 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.928589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[1, 0, 0, 27413, 32183, 5932, 6, 1] avg=8192.0000 max=32183 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.928589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[1, 0, 0, 27413, 32183, 5932, 6, 1] avg=8192.0000 max=32183 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.928589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[1, 0, 0, 27413, 32183, 5932, 6, 1] avg=8192.0000 max=32183 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.928589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[8273, 8276, 8275, 8275, 8272, 7949, 7944, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[8273, 8276, 8275, 8275, 8272, 7949, 7944, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[6396, 8596, 8596, 8597, 8596, 7564, 8595, 8596] avg=8192.0000 max=8597 max_rank=3 min=6396 min_rank=0 imbalance_ratio=1.049438 max_load_reduction=23586 max_load_reduction_pct=0.732871 moved_to_replicas=86318 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[6396, 8596, 8596, 8597, 8596, 7564, 8595, 8596] avg=8192.0000 max=8597 max_rank=3 min=6396 min_rank=0 imbalance_ratio=1.049438 max_load_reduction=23586 max_load_reduction_pct=0.732871 moved_to_replicas=86318 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[1, 0, 0, 27413, 32183, 5932, 6, 1] avg=8192.0000 max=32183 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.928589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[6396, 8596, 8596, 8597, 8596, 7564, 8595, 8596] avg=8192.0000 max=8597 max_rank=3 min=6396 min_rank=0 imbalance_ratio=1.049438 max_load_reduction=23586 max_load_reduction_pct=0.732871 moved_to_replicas=86318 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[6396, 8596, 8596, 8597, 8596, 7564, 8595, 8596] avg=8192.0000 max=8597 max_rank=3 min=6396 min_rank=0 imbalance_ratio=1.049438 max_load_reduction=23586 max_load_reduction_pct=0.732871 moved_to_replicas=86318 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[6396, 8596, 8596, 8597, 8596, 7564, 8595, 8596] avg=8192.0000 max=8597 max_rank=3 min=6396 min_rank=0 imbalance_ratio=1.049438 max_load_reduction=23586 max_load_reduction_pct=0.732871 moved_to_replicas=86318 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[6396, 8596, 8596, 8597, 8596, 7564, 8595, 8596] avg=8192.0000 max=8597 max_rank=3 min=6396 min_rank=0 imbalance_ratio=1.049438 max_load_reduction=23586 max_load_reduction_pct=0.732871 moved_to_replicas=86318 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[1, 0, 0, 27413, 32183, 5932, 6, 1] avg=8192.0000 max=32183 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.928589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[1, 0, 0, 27413, 32183, 5932, 6, 1] avg=8192.0000 max=32183 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.928589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[6396, 8596, 8596, 8597, 8596, 7564, 8595, 8596] avg=8192.0000 max=8597 max_rank=3 min=6396 min_rank=0 imbalance_ratio=1.049438 max_load_reduction=23586 max_load_reduction_pct=0.732871 moved_to_replicas=86318 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[6396, 8596, 8596, 8597, 8596, 7564, 8595, 8596] avg=8192.0000 max=8597 max_rank=3 min=6396 min_rank=0 imbalance_ratio=1.049438 max_load_reduction=23586 max_load_reduction_pct=0.732871 moved_to_replicas=86318 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[8273, 8276, 8275, 8275, 8272, 7944, 7949, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[8273, 8276, 8275, 8275, 8272, 7944, 7949, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[8273, 8276, 8275, 8275, 8272, 7944, 7949, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[8273, 8276, 8275, 8275, 8272, 7944, 7949, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[8273, 8276, 8275, 8275, 8272, 7944, 7949, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[8273, 8276, 8275, 8275, 8272, 7944, 7949, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[0, 0, 1, 34, 0, 32767, 1, 32733] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[0, 0, 1, 34, 0, 32767, 1, 32733] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[0, 0, 1, 34, 0, 32767, 1, 32733] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[0, 0, 1, 34, 0, 32767, 1, 32733] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[0, 0, 1, 34, 0, 32767, 1, 32733] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[8273, 8276, 8275, 8275, 8272, 7944, 7949, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[8273, 8276, 8275, 8275, 8272, 7944, 7949, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[8276, 8275, 8273, 7947, 7944, 8272, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97910 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[8276, 8275, 8273, 7947, 7944, 8272, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97910 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[0, 0, 1, 34, 0, 32767, 1, 32733] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[8276, 8275, 8273, 7947, 7944, 8272, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97910 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[8276, 8275, 8273, 7947, 7944, 8272, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97910 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[8276, 8275, 8273, 7947, 7944, 8272, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97910 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[8276, 8275, 8273, 7947, 7944, 8272, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97910 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[0, 0, 1, 34, 0, 32767, 1, 32733] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[1, 32765, 0, 1, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[0, 0, 1, 34, 0, 32767, 1, 32733] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[1, 32765, 0, 1, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[1, 32765, 0, 1, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[1, 32765, 0, 1, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[1, 32765, 0, 1, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[8276, 8275, 8273, 7947, 7944, 8272, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97910 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[7945, 8275, 8276, 8275, 8275, 8272, 8273, 7945] avg=8192.0000 max=8276 max_rank=2 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97969 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[8276, 8275, 8273, 7947, 7944, 8272, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97910 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[1, 32765, 0, 1, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[7945, 8275, 8276, 8275, 8275, 8272, 8273, 7945] avg=8192.0000 max=8276 max_rank=2 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97969 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[7945, 8275, 8276, 8275, 8275, 8272, 8273, 7945] avg=8192.0000 max=8276 max_rank=2 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97969 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[7945, 8275, 8276, 8275, 8275, 8272, 8273, 7945] avg=8192.0000 max=8276 max_rank=2 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97969 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[7945, 8275, 8276, 8275, 8275, 8272, 8273, 7945] avg=8192.0000 max=8276 max_rank=2 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97969 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[7945, 8275, 8276, 8275, 8275, 8272, 8273, 7945] avg=8192.0000 max=8276 max_rank=2 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97969 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[1, 32765, 0, 1, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[32767, 0, 0, 0, 1, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[32767, 0, 0, 0, 1, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[1, 32765, 0, 1, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[32767, 0, 0, 0, 1, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[32767, 0, 0, 0, 1, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[32767, 0, 0, 0, 1, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[7945, 8275, 8276, 8275, 8275, 8272, 8273, 7945] avg=8192.0000 max=8276 max_rank=2 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97969 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[8276, 8276, 8272, 7943, 8273, 8275, 7946, 8275] avg=8192.0000 max=8276 max_rank=0 min=7943 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[8276, 8276, 8272, 7943, 8273, 8275, 7946, 8275] avg=8192.0000 max=8276 max_rank=0 min=7943 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[7945, 8275, 8276, 8275, 8275, 8272, 8273, 7945] avg=8192.0000 max=8276 max_rank=2 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97969 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[32767, 0, 0, 0, 1, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[8276, 8276, 8272, 7943, 8273, 8275, 7946, 8275] avg=8192.0000 max=8276 max_rank=0 min=7943 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[8276, 8276, 8272, 7943, 8273, 8275, 7946, 8275] avg=8192.0000 max=8276 max_rank=0 min=7943 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[8276, 8276, 8272, 7943, 8273, 8275, 7946, 8275] avg=8192.0000 max=8276 max_rank=0 min=7943 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[8276, 8276, 8272, 7943, 8273, 8275, 7946, 8275] avg=8192.0000 max=8276 max_rank=0 min=7943 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[1, 21, 0, 32746, 1, 0, 0, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[32767, 0, 0, 0, 1, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[1, 21, 0, 32746, 1, 0, 0, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[32767, 0, 0, 0, 1, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[1, 21, 0, 32746, 1, 0, 0, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[1, 21, 0, 32746, 1, 0, 0, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[1, 21, 0, 32746, 1, 0, 0, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[8274, 7947, 8276, 8275, 8273, 8275, 7944, 8272] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97932 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[8276, 8276, 8272, 7943, 8273, 8275, 7946, 8275] avg=8192.0000 max=8276 max_rank=0 min=7943 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[8274, 7947, 8276, 8275, 8273, 8275, 7944, 8272] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97932 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[1, 21, 0, 32746, 1, 0, 0, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[8274, 7947, 8276, 8275, 8273, 8275, 7944, 8272] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97932 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[8276, 8276, 8272, 7943, 8273, 8275, 7946, 8275] avg=8192.0000 max=8276 max_rank=0 min=7943 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[8274, 7947, 8276, 8275, 8273, 8275, 7944, 8272] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97932 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[8274, 7947, 8276, 8275, 8273, 8275, 7944, 8272] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97932 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[10, 0, 1, 1, 32757, 32767, 0, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[8274, 7947, 8276, 8275, 8273, 8275, 7944, 8272] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97932 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[1, 21, 0, 32746, 1, 0, 0, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[10, 0, 1, 1, 32757, 32767, 0, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[10, 0, 1, 1, 32757, 32767, 0, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[1, 21, 0, 32746, 1, 0, 0, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[10, 0, 1, 1, 32757, 32767, 0, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[10, 0, 1, 1, 32757, 32767, 0, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[7947, 8276, 8273, 8274, 8275, 8275, 8272, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97958 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[7947, 8276, 8273, 8274, 8275, 8275, 8272, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97958 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[8274, 7947, 8276, 8275, 8273, 8275, 7944, 8272] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97932 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[7947, 8276, 8273, 8274, 8275, 8275, 8272, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97958 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[8274, 7947, 8276, 8275, 8273, 8275, 7944, 8272] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97932 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[10, 0, 1, 1, 32757, 32767, 0, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[7947, 8276, 8273, 8274, 8275, 8275, 8272, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97958 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[7947, 8276, 8273, 8274, 8275, 8275, 8272, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97958 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[1, 32767, 12, 1, 32747, 0, 8, 0] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=5 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[1, 32767, 12, 1, 32747, 0, 8, 0] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=5 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[7947, 8276, 8273, 8274, 8275, 8275, 8272, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97958 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[10, 0, 1, 1, 32757, 32767, 0, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[1, 32767, 12, 1, 32747, 0, 8, 0] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=5 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[10, 0, 1, 1, 32757, 32767, 0, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[1, 32767, 12, 1, 32747, 0, 8, 0] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=5 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[1, 32767, 12, 1, 32747, 0, 8, 0] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=5 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[7945, 8276, 7946, 8274, 8274, 8275, 8274, 8272] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[7945, 8276, 7946, 8274, 8274, 8275, 8274, 8272] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[7947, 8276, 8273, 8274, 8275, 8275, 8272, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97958 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[7945, 8276, 7946, 8274, 8274, 8275, 8274, 8272] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[1, 32767, 12, 1, 32747, 0, 8, 0] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=5 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[7947, 8276, 8273, 8274, 8275, 8275, 8272, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97958 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[7945, 8276, 7946, 8274, 8274, 8275, 8274, 8272] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[7945, 8276, 7946, 8274, 8274, 8275, 8274, 8272] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[0, 1, 1, 0, 0, 0, 32767, 32767] avg=8192.0000 max=32767 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[0, 1, 1, 0, 0, 0, 32767, 32767] avg=8192.0000 max=32767 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[7945, 8276, 7946, 8274, 8274, 8275, 8274, 8272] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[1, 32767, 12, 1, 32747, 0, 8, 0] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=5 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[0, 1, 1, 0, 0, 0, 32767, 32767] avg=8192.0000 max=32767 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[0, 1, 1, 0, 0, 0, 32767, 32767] avg=8192.0000 max=32767 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[1, 32767, 12, 1, 32747, 0, 8, 0] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=5 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[8276, 8273, 7946, 8275, 7944, 8275, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[0, 1, 1, 0, 0, 0, 32767, 32767] avg=8192.0000 max=32767 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[8276, 8273, 7946, 8275, 7944, 8275, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[7945, 8276, 7946, 8274, 8274, 8275, 8274, 8272] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[8276, 8273, 7946, 8275, 7944, 8275, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[0, 1, 1, 0, 0, 0, 32767, 32767] avg=8192.0000 max=32767 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[8276, 8273, 7946, 8275, 7944, 8275, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[7945, 8276, 7946, 8274, 8274, 8275, 8274, 8272] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[8276, 8273, 7946, 8275, 7944, 8275, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[6, 0, 32767, 1, 31355, 1406, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[6, 0, 32767, 1, 31355, 1406, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[8276, 8273, 7946, 8275, 7944, 8275, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[6, 0, 32767, 1, 31355, 1406, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[0, 1, 1, 0, 0, 0, 32767, 32767] avg=8192.0000 max=32767 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[6, 0, 32767, 1, 31355, 1406, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[8274, 8276, 8276, 7944, 8274, 7945, 8272, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94992 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[0, 1, 1, 0, 0, 0, 32767, 32767] avg=8192.0000 max=32767 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[8274, 8276, 8276, 7944, 8274, 7945, 8272, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94992 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[6, 0, 32767, 1, 31355, 1406, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[8274, 8276, 8276, 7944, 8274, 7945, 8272, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94992 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[8276, 8273, 7946, 8275, 7944, 8275, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[6, 0, 32767, 1, 31355, 1406, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[8274, 8276, 8276, 7944, 8274, 7945, 8272, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94992 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[8276, 8273, 7946, 8275, 7944, 8275, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[8274, 8276, 8276, 7944, 8274, 7945, 8272, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94992 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[32767, 0, 1, 2, 56, 0, 32706, 4] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[32767, 0, 1, 2, 56, 0, 32706, 4] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[8274, 8276, 8276, 7944, 8274, 7945, 8272, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94992 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[32767, 0, 1, 2, 56, 0, 32706, 4] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[6, 0, 32767, 1, 31355, 1406, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[32767, 0, 1, 2, 56, 0, 32706, 4] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[8276, 8275, 7945, 8274, 7946, 8272, 8274, 8274] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97846 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[6, 0, 32767, 1, 31355, 1406, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[8276, 8275, 7945, 8274, 7946, 8272, 8274, 8274] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97846 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[32767, 0, 1, 2, 56, 0, 32706, 4] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[8276, 8275, 7945, 8274, 7946, 8272, 8274, 8274] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97846 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[8274, 8276, 8276, 7944, 8274, 7945, 8272, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94992 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[32767, 0, 1, 2, 56, 0, 32706, 4] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[8276, 8275, 7945, 8274, 7946, 8272, 8274, 8274] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97846 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[8274, 8276, 8276, 7944, 8274, 7945, 8272, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94992 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[8276, 8275, 7945, 8274, 7946, 8272, 8274, 8274] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97846 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[0, 0, 32668, 138, 1, 1, 0, 32728] avg=8192.0000 max=32728 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.995117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[0, 0, 32668, 138, 1, 1, 0, 32728] avg=8192.0000 max=32728 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.995117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[8276, 8275, 7945, 8274, 7946, 8272, 8274, 8274] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97846 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[0, 0, 32668, 138, 1, 1, 0, 32728] avg=8192.0000 max=32728 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.995117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[32767, 0, 1, 2, 56, 0, 32706, 4] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[8274, 8274, 8274, 7986, 8274, 8274, 7906, 8274] avg=8192.0000 max=8274 max_rank=0 min=7906 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24454 max_load_reduction_pct=0.747189 moved_to_replicas=97756 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[0, 0, 32668, 138, 1, 1, 0, 32728] avg=8192.0000 max=32728 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.995117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[8274, 8274, 8274, 7986, 8274, 8274, 7906, 8274] avg=8192.0000 max=8274 max_rank=0 min=7906 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24454 max_load_reduction_pct=0.747189 moved_to_replicas=97756 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[32767, 0, 1, 2, 56, 0, 32706, 4] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[0, 0, 32668, 138, 1, 1, 0, 32728] avg=8192.0000 max=32728 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.995117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[8274, 8274, 8274, 7986, 8274, 8274, 7906, 8274] avg=8192.0000 max=8274 max_rank=0 min=7906 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24454 max_load_reduction_pct=0.747189 moved_to_replicas=97756 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[8276, 8275, 7945, 8274, 7946, 8272, 8274, 8274] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97846 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[0, 0, 32668, 138, 1, 1, 0, 32728] avg=8192.0000 max=32728 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.995117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[8274, 8274, 8274, 7986, 8274, 8274, 7906, 8274] avg=8192.0000 max=8274 max_rank=0 min=7906 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24454 max_load_reduction_pct=0.747189 moved_to_replicas=97756 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[8276, 8275, 7945, 8274, 7946, 8272, 8274, 8274] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97846 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[8274, 8274, 8274, 7986, 8274, 8274, 7906, 8274] avg=8192.0000 max=8274 max_rank=0 min=7906 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24454 max_load_reduction_pct=0.747189 moved_to_replicas=97756 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[1, 61, 0, 0, 32767, 0, 1, 32706] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[1, 61, 0, 0, 32767, 0, 1, 32706] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[8274, 8274, 8274, 7986, 8274, 8274, 7906, 8274] avg=8192.0000 max=8274 max_rank=0 min=7906 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24454 max_load_reduction_pct=0.747189 moved_to_replicas=97756 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[1, 61, 0, 0, 32767, 0, 1, 32706] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[0, 0, 32668, 138, 1, 1, 0, 32728] avg=8192.0000 max=32728 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.995117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[8273, 7947, 8276, 8275, 8272, 7944, 8274, 8275] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97855 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[1, 61, 0, 0, 32767, 0, 1, 32706] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[8273, 7947, 8276, 8275, 8272, 7944, 8274, 8275] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97855 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[0, 0, 32668, 138, 1, 1, 0, 32728] avg=8192.0000 max=32728 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.995117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[1, 61, 0, 0, 32767, 0, 1, 32706] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[8273, 7947, 8276, 8275, 8272, 7944, 8274, 8275] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97855 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[8274, 8274, 8274, 7986, 8274, 8274, 7906, 8274] avg=8192.0000 max=8274 max_rank=0 min=7906 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24454 max_load_reduction_pct=0.747189 moved_to_replicas=97756 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[1, 61, 0, 0, 32767, 0, 1, 32706] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[8273, 7947, 8276, 8275, 8272, 7944, 8274, 8275] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97855 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[8274, 8274, 8274, 7986, 8274, 8274, 7906, 8274] avg=8192.0000 max=8274 max_rank=0 min=7906 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24454 max_load_reduction_pct=0.747189 moved_to_replicas=97756 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[8273, 7947, 8276, 8275, 8272, 7944, 8274, 8275] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97855 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[8273, 7947, 8276, 8275, 8272, 7944, 8274, 8275] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97855 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[1, 61, 0, 0, 32767, 0, 1, 32706] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[1, 61, 0, 0, 32767, 0, 1, 32706] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[8273, 7947, 8276, 8275, 8272, 7944, 8274, 8275] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97855 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[8273, 7947, 8276, 8275, 8272, 7944, 8274, 8275] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97855 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=32 total_expected_recv=267421 total_dropped_tokens=678 drop_rate_pct=0.253533 records_with_drop_pct=3.125000 peak_rank_overflow=678 total_slack_slots=58240 peak_demand=10918 required_capacity=1.332764 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=32 total_expected_recv=268532 total_dropped_tokens=3079 drop_rate_pct=1.146605 records_with_drop_pct=6.250000 peak_rank_overflow=1767 total_slack_slots=59136 peak_demand=12007 required_capacity=1.465698 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=32 total_expected_recv=268806 total_dropped_tokens=3081 drop_rate_pct=1.146180 records_with_drop_pct=6.250000 peak_rank_overflow=1769 total_slack_slots=59264 peak_demand=12009 required_capacity=1.465942 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=32 total_expected_recv=251622 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=73472 peak_demand=8276 required_capacity=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=32 total_expected_recv=255065 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=70400 peak_demand=8596 required_capacity=1.049316 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=32 total_expected_recv=270075 total_dropped_tokens=3079 drop_rate_pct=1.140054 records_with_drop_pct=6.250000 peak_rank_overflow=1767 total_slack_slots=57600 peak_demand=12007 required_capacity=1.465698 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=32 total_expected_recv=246078 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78464 peak_demand=8276 required_capacity=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=32 total_expected_recv=269553 total_dropped_tokens=3080 drop_rate_pct=1.142632 records_with_drop_pct=6.250000 peak_rank_overflow=1768 total_slack_slots=58752 peak_demand=12008 required_capacity=1.465820 + [2026-08-30 05:08:04] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 887.2 | throughput per GPU (TFLOP/s/GPU): 375.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.453164E+00 | loss scale: 1.0 | grad norm: 6.208 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 10240 | min safe capacity: 1.2500 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=10240 expected_recv=18463 expected_overflow=0 exact_recv_per_rank=[9203, 5171, 9202, 9203, 9203, 9202, 8071, 6281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=10240 expected_recv=10895 expected_overflow=0 exact_recv_per_rank=[9203, 5171, 9202, 9203, 9203, 9202, 8071, 6281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=10240 expected_recv=18461 expected_overflow=0 exact_recv_per_rank=[9203, 5171, 9202, 9203, 9203, 9202, 8071, 6281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=10240 expected_recv=14915 expected_overflow=0 exact_recv_per_rank=[9203, 5171, 9202, 9203, 9203, 9202, 8071, 6281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=10240 expected_recv=18461 expected_overflow=0 exact_recv_per_rank=[9203, 5171, 9202, 9203, 9203, 9202, 8071, 6281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=10240 expected_recv=18462 expected_overflow=0 exact_recv_per_rank=[9203, 5171, 9202, 9203, 9203, 9202, 8071, 6281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=10240 expected_recv=12953 expected_overflow=0 exact_recv_per_rank=[9203, 5171, 9202, 9203, 9203, 9202, 8071, 6281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=10240 expected_recv=18461 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=10240 expected_recv=18463 actual_valid_recv=18688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=10240 expected_recv=10895 actual_valid_recv=11264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=10240 expected_recv=18462 expected_overflow=0 exact_recv_per_rank=[9203, 5171, 9202, 9203, 9203, 9202, 8071, 6281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=10240 expected_recv=14915 actual_valid_recv=15232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=10240 expected_recv=18461 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=18560 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=18688 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=11264 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=10240 expected_recv=18462 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=10240 expected_recv=12953 actual_valid_recv=13440 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=15232 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=18560 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=10240 expected_recv=18462 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=13440 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=18560 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=18560 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=10240 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8272, 8275, 7947, 8275, 7944, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8272, 8275, 7947, 8275, 7944, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8272, 8275, 7947, 8275, 7944, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=10240 expected_recv=16545 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=10240 expected_recv=15884 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8272, 8275, 7947, 8275, 7944, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8272, 8275, 7947, 8275, 7944, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8272, 8275, 7947, 8275, 7944, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=10240 expected_recv=15884 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8272, 8275, 7947, 8275, 7944, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=10240 expected_recv=15900 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8272, 8275, 7947, 8275, 7944, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=10240 expected_recv=15900 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8275, 8273, 8272, 7946, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8275, 8273, 8272, 7946, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=10240 expected_recv=16222 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8275, 8273, 8272, 7946, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=10240 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8275, 8273, 8272, 7946, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=10240 expected_recv=16222 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8275, 8273, 8272, 7946, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=10240 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8275, 8273, 8272, 7946, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=10240 expected_recv=16220 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8275, 8273, 8272, 7946, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=10240 expected_recv=15888 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=10240 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8275, 8273, 8272, 7946, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=10240 expected_recv=16545 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=10240 expected_recv=18014 expected_overflow=0 exact_recv_per_rank=[3875, 8885, 8886, 8887, 8601, 8886, 8885, 8631] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=10240 expected_recv=18994 expected_overflow=0 exact_recv_per_rank=[3875, 8885, 8886, 8887, 8601, 8886, 8885, 8631] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=10240 expected_recv=18014 actual_valid_recv=18304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=10240 expected_recv=18994 actual_valid_recv=19072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=10240 expected_recv=9745 expected_overflow=0 exact_recv_per_rank=[3875, 8885, 8886, 8887, 8601, 8886, 8885, 8631] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=10240 expected_recv=18992 expected_overflow=0 exact_recv_per_rank=[3875, 8885, 8886, 8887, 8601, 8886, 8885, 8631] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=18304 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=19072 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=10240 expected_recv=18993 expected_overflow=0 exact_recv_per_rank=[3875, 8885, 8886, 8887, 8601, 8886, 8885, 8631] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=10240 expected_recv=9745 actual_valid_recv=10112 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=10240 expected_recv=8634 expected_overflow=0 exact_recv_per_rank=[3875, 8885, 8886, 8887, 8601, 8886, 8885, 8631] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=10240 expected_recv=18992 actual_valid_recv=19072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=10240 expected_recv=18993 actual_valid_recv=19072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=10112 slack_slots=10368 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=10240 expected_recv=18993 expected_overflow=0 exact_recv_per_rank=[3875, 8885, 8886, 8887, 8601, 8886, 8885, 8631] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=10240 expected_recv=8634 actual_valid_recv=8960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=10240 expected_recv=18707 expected_overflow=0 exact_recv_per_rank=[3875, 8885, 8886, 8887, 8601, 8886, 8885, 8631] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=19072 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=19072 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=8960 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=10240 expected_recv=18993 actual_valid_recv=19072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=10240 expected_recv=18707 actual_valid_recv=18944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=19072 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=18944 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8276, 8272, 7944, 8272, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8276, 8272, 7944, 8272, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=10240 expected_recv=16226 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8276, 8272, 7944, 8272, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=10240 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8276, 8272, 7944, 8272, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8276, 8272, 7944, 8272, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8276, 8272, 7944, 8272, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=10240 expected_recv=16226 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=10240 expected_recv=16216 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8276, 8272, 7944, 8272, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8276, 8272, 7944, 8272, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 8273, 8276, 8272, 7944, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 8273, 8276, 8272, 7944, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 8273, 8276, 8272, 7944, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 8273, 8276, 8272, 7944, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 8273, 8276, 8272, 7944, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=10240 expected_recv=15894 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 8273, 8276, 8272, 7944, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=10240 expected_recv=15894 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 8273, 8276, 8272, 7944, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 8273, 8276, 8272, 7944, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[7947, 8276, 8275, 8275, 8272, 7944, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=10240 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[7947, 8276, 8275, 8275, 8272, 7944, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=10240 expected_recv=16220 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=10240 expected_recv=15894 expected_overflow=0 exact_recv_per_rank=[7947, 8276, 8275, 8275, 8272, 7944, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7947, 8276, 8275, 8275, 8272, 7944, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=10240 expected_recv=15894 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[7947, 8276, 8275, 8275, 8272, 7944, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7947, 8276, 8275, 8275, 8272, 7944, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=10240 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[7947, 8276, 8275, 8275, 8272, 7944, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[7947, 8276, 8275, 8275, 8272, 7944, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 7944, 8273, 8275, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=10240 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=10240 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 7944, 8273, 8275, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=10240 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 7944, 8273, 8275, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=10240 expected_recv=15893 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 7944, 8273, 8275, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 7944, 8273, 8275, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=10240 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 7944, 8273, 8275, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=10240 expected_recv=15893 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 7944, 8273, 8275, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=10240 expected_recv=15888 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 7944, 8273, 8275, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=10240 expected_recv=15894 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8276, 8275, 8273, 8275, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=10240 expected_recv=15894 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8276, 8275, 8273, 8275, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=10240 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8276, 8275, 8273, 8275, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=10240 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8276, 8275, 8273, 8275, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8276, 8275, 8273, 8275, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=10240 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8276, 8275, 8273, 8275, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=10240 expected_recv=16218 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=10240 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=10240 expected_recv=16220 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8276, 8275, 8273, 8275, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8276, 8275, 8273, 8275, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=10240 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8273, 7946, 8275, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=10240 expected_recv=16220 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8273, 7946, 8275, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8273, 7946, 8275, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=10240 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8273, 7946, 8275, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8273, 7946, 8275, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=10240 expected_recv=16216 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8273, 7946, 8275, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=10240 expected_recv=15894 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8273, 7946, 8275, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8273, 7946, 8275, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8276, 7945, 8275, 8272, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=10240 expected_recv=15894 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8276, 7945, 8275, 8272, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=10240 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8276, 7945, 8275, 8272, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=10240 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8276, 7945, 8275, 8272, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=10240 expected_recv=15895 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8276, 7945, 8275, 8272, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=10240 expected_recv=16217 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8276, 7945, 8275, 8272, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=10240 expected_recv=16216 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=10240 expected_recv=15895 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8276, 7945, 8275, 8272, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=10240 expected_recv=15891 expected_overflow=0 exact_recv_per_rank=[8273, 7945, 8275, 8276, 8275, 7947, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8276, 7945, 8275, 8272, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=10240 expected_recv=15891 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8273, 7945, 8275, 8276, 8275, 7947, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=10240 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8273, 7945, 8275, 8276, 8275, 7947, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=10240 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8273, 7945, 8275, 8276, 8275, 7947, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8273, 7945, 8275, 8276, 8275, 7947, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=10240 expected_recv=16220 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=10240 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 7945, 8275, 8276, 8275, 7947, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 7945, 7947, 8273, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8273, 7945, 8275, 8276, 8275, 7947, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8273, 7945, 8275, 8276, 8275, 7947, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 7945, 7947, 8273, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=10240 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 7945, 7947, 8273, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 7945, 7947, 8273, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=10240 expected_recv=15892 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 7945, 7947, 8273, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=10240 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=10240 expected_recv=15892 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 7945, 7947, 8273, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 7945, 8275, 7946, 8272, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=10240 expected_recv=15893 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 7945, 7947, 8273, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 7945, 7947, 8273, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=10240 expected_recv=15893 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 7945, 8275, 7946, 8272, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 7945, 8275, 7946, 8272, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 7945, 8275, 7946, 8272, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 7945, 8275, 7946, 8272, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 7945, 8275, 7946, 8272, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=10240 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8276, 8272, 7945, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=10240 expected_recv=15894 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 7945, 8275, 7946, 8272, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 7945, 8275, 7946, 8272, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=10240 expected_recv=16220 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=10240 expected_recv=15894 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8276, 8272, 7945, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8276, 8272, 7945, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=10240 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8276, 8272, 7945, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8276, 8272, 7945, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=10240 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8276, 8272, 7945, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=10240 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8276, 8272, 7945, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8276, 8275, 8272, 7944, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=10240 expected_recv=15893 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8276, 8272, 7945, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=10240 expected_recv=16545 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=10240 expected_recv=15893 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=10240 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8276, 8275, 8272, 7944, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=10240 expected_recv=15894 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8276, 8275, 8272, 7944, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8276, 8275, 8272, 7944, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8276, 8275, 8272, 7944, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=10240 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=10240 expected_recv=15894 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8276, 8275, 8272, 7944, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=10240 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8276, 8275, 8272, 7944, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=10240 expected_recv=15886 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8276, 8275, 8272, 7944, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[4952, 38, 17273, 15483, 14336, 13453, 0, 1] avg=8192.0000 max=17273 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.108521 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=10240 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=10240 expected_recv=15886 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[4952, 38, 17273, 15483, 14336, 13453, 0, 1] avg=8192.0000 max=17273 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.108521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[4952, 38, 17273, 15483, 14336, 13453, 0, 1] avg=8192.0000 max=17273 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.108521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[4952, 38, 17273, 15483, 14336, 13453, 0, 1] avg=8192.0000 max=17273 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.108521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[4952, 38, 17273, 15483, 14336, 13453, 0, 1] avg=8192.0000 max=17273 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.108521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[9203, 5171, 9202, 9203, 9203, 9202, 8071, 6281] avg=8192.0000 max=9203 max_rank=0 min=5171 min_rank=1 imbalance_ratio=1.123413 max_load_reduction=8070 max_load_reduction_pct=0.467203 moved_to_replicas=48568 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[9203, 5171, 9202, 9203, 9203, 9202, 8071, 6281] avg=8192.0000 max=9203 max_rank=0 min=5171 min_rank=1 imbalance_ratio=1.123413 max_load_reduction=8070 max_load_reduction_pct=0.467203 moved_to_replicas=48568 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[9203, 5171, 9202, 9203, 9203, 9202, 8071, 6281] avg=8192.0000 max=9203 max_rank=0 min=5171 min_rank=1 imbalance_ratio=1.123413 max_load_reduction=8070 max_load_reduction_pct=0.467203 moved_to_replicas=48568 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[9203, 5171, 9202, 9203, 9203, 9202, 8071, 6281] avg=8192.0000 max=9203 max_rank=0 min=5171 min_rank=1 imbalance_ratio=1.123413 max_load_reduction=8070 max_load_reduction_pct=0.467203 moved_to_replicas=48568 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[9203, 5171, 9202, 9203, 9203, 9202, 8071, 6281] avg=8192.0000 max=9203 max_rank=0 min=5171 min_rank=1 imbalance_ratio=1.123413 max_load_reduction=8070 max_load_reduction_pct=0.467203 moved_to_replicas=48568 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[4952, 38, 17273, 15483, 14336, 13453, 0, 1] avg=8192.0000 max=17273 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.108521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[4952, 38, 17273, 15483, 14336, 13453, 0, 1] avg=8192.0000 max=17273 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.108521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[4952, 38, 17273, 15483, 14336, 13453, 0, 1] avg=8192.0000 max=17273 max_rank=2 min=0 min_rank=6 imbalance_ratio=2.108521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[0, 32734, 32766, 1, 34, 0, 0, 1] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[9203, 5171, 9202, 9203, 9203, 9202, 8071, 6281] avg=8192.0000 max=9203 max_rank=0 min=5171 min_rank=1 imbalance_ratio=1.123413 max_load_reduction=8070 max_load_reduction_pct=0.467203 moved_to_replicas=48568 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[9203, 5171, 9202, 9203, 9203, 9202, 8071, 6281] avg=8192.0000 max=9203 max_rank=0 min=5171 min_rank=1 imbalance_ratio=1.123413 max_load_reduction=8070 max_load_reduction_pct=0.467203 moved_to_replicas=48568 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[0, 32734, 32766, 1, 34, 0, 0, 1] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[0, 32734, 32766, 1, 34, 0, 0, 1] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[9203, 5171, 9202, 9203, 9203, 9202, 8071, 6281] avg=8192.0000 max=9203 max_rank=0 min=5171 min_rank=1 imbalance_ratio=1.123413 max_load_reduction=8070 max_load_reduction_pct=0.467203 moved_to_replicas=48568 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[0, 32734, 32766, 1, 34, 0, 0, 1] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[8275, 8273, 8272, 8275, 7947, 8275, 7944, 8275] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97905 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[0, 32734, 32766, 1, 34, 0, 0, 1] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[8275, 8273, 8272, 8275, 7947, 8275, 7944, 8275] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97905 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[8275, 8273, 8272, 8275, 7947, 8275, 7944, 8275] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97905 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[8275, 8273, 8272, 8275, 7947, 8275, 7944, 8275] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97905 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[8275, 8273, 8272, 8275, 7947, 8275, 7944, 8275] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97905 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[0, 32734, 32766, 1, 34, 0, 0, 1] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[0, 32734, 32766, 1, 34, 0, 0, 1] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[1, 0, 32767, 32766, 1, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[0, 32734, 32766, 1, 34, 0, 0, 1] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[1, 0, 32767, 32766, 1, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[1, 0, 32767, 32766, 1, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[8275, 8273, 8272, 8275, 7947, 8275, 7944, 8275] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97905 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[8275, 8273, 8272, 8275, 7947, 8275, 7944, 8275] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97905 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[1, 0, 32767, 32766, 1, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[8275, 8276, 8275, 8275, 8273, 8272, 7946, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[8275, 8273, 8272, 8275, 7947, 8275, 7944, 8275] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97905 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[1, 0, 32767, 32766, 1, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[8275, 8276, 8275, 8275, 8273, 8272, 7946, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[8275, 8276, 8275, 8275, 8273, 8272, 7946, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[8275, 8276, 8275, 8275, 8273, 8272, 7946, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[8275, 8276, 8275, 8275, 8273, 8272, 7946, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[1, 0, 32767, 32766, 1, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[1, 0, 32767, 32766, 1, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[1, 0, 0, 30532, 3, 17484, 17515, 1] avg=8192.0000 max=30532 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.727051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[1, 0, 32767, 32766, 1, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[1, 0, 0, 30532, 3, 17484, 17515, 1] avg=8192.0000 max=30532 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.727051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[1, 0, 0, 30532, 3, 17484, 17515, 1] avg=8192.0000 max=30532 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.727051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[8275, 8276, 8275, 8275, 8273, 8272, 7946, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[1, 0, 0, 30532, 3, 17484, 17515, 1] avg=8192.0000 max=30532 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.727051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[3875, 8885, 8886, 8887, 8601, 8886, 8885, 8631] avg=8192.0000 max=8887 max_rank=3 min=3875 min_rank=0 imbalance_ratio=1.084839 max_load_reduction=21645 max_load_reduction_pct=0.708928 moved_to_replicas=74082 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[8275, 8276, 8275, 8275, 8273, 8272, 7946, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[8275, 8276, 8275, 8275, 8273, 8272, 7946, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[1, 0, 0, 30532, 3, 17484, 17515, 1] avg=8192.0000 max=30532 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.727051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[3875, 8885, 8886, 8887, 8601, 8886, 8885, 8631] avg=8192.0000 max=8887 max_rank=3 min=3875 min_rank=0 imbalance_ratio=1.084839 max_load_reduction=21645 max_load_reduction_pct=0.708928 moved_to_replicas=74082 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[3875, 8885, 8886, 8887, 8601, 8886, 8885, 8631] avg=8192.0000 max=8887 max_rank=3 min=3875 min_rank=0 imbalance_ratio=1.084839 max_load_reduction=21645 max_load_reduction_pct=0.708928 moved_to_replicas=74082 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[3875, 8885, 8886, 8887, 8601, 8886, 8885, 8631] avg=8192.0000 max=8887 max_rank=3 min=3875 min_rank=0 imbalance_ratio=1.084839 max_load_reduction=21645 max_load_reduction_pct=0.708928 moved_to_replicas=74082 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 0, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[3875, 8885, 8886, 8887, 8601, 8886, 8885, 8631] avg=8192.0000 max=8887 max_rank=3 min=3875 min_rank=0 imbalance_ratio=1.084839 max_load_reduction=21645 max_load_reduction_pct=0.708928 moved_to_replicas=74082 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[1, 0, 0, 30532, 3, 17484, 17515, 1] avg=8192.0000 max=30532 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.727051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[1, 0, 0, 30532, 3, 17484, 17515, 1] avg=8192.0000 max=30532 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.727051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[1, 0, 0, 30532, 3, 17484, 17515, 1] avg=8192.0000 max=30532 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.727051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 0, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 0, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 0, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[8276, 8276, 8275, 8276, 8272, 7944, 8272, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97968 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[3875, 8885, 8886, 8887, 8601, 8886, 8885, 8631] avg=8192.0000 max=8887 max_rank=3 min=3875 min_rank=0 imbalance_ratio=1.084839 max_load_reduction=21645 max_load_reduction_pct=0.708928 moved_to_replicas=74082 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[3875, 8885, 8886, 8887, 8601, 8886, 8885, 8631] avg=8192.0000 max=8887 max_rank=3 min=3875 min_rank=0 imbalance_ratio=1.084839 max_load_reduction=21645 max_load_reduction_pct=0.708928 moved_to_replicas=74082 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[3875, 8885, 8886, 8887, 8601, 8886, 8885, 8631] avg=8192.0000 max=8887 max_rank=3 min=3875 min_rank=0 imbalance_ratio=1.084839 max_load_reduction=21645 max_load_reduction_pct=0.708928 moved_to_replicas=74082 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[8276, 8276, 8275, 8276, 8272, 7944, 8272, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97968 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 0, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[8276, 8276, 8275, 8276, 8272, 7944, 8272, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97968 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[8276, 8276, 8275, 8276, 8272, 7944, 8272, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97968 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[0, 1, 1, 32759, 0, 32767, 0, 8] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[8276, 8276, 8275, 8276, 8272, 7944, 8272, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97968 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 0, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 0, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 0, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[0, 1, 1, 32759, 0, 32767, 0, 8] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[0, 1, 1, 32759, 0, 32767, 0, 8] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[0, 1, 1, 32759, 0, 32767, 0, 8] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[8275, 8274, 8275, 8273, 8276, 8272, 7944, 7947] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97879 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[8276, 8276, 8275, 8276, 8272, 7944, 8272, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97968 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[8276, 8276, 8275, 8276, 8272, 7944, 8272, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97968 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[8276, 8276, 8275, 8276, 8272, 7944, 8272, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97968 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[8275, 8274, 8275, 8273, 8276, 8272, 7944, 7947] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97879 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[8275, 8274, 8275, 8273, 8276, 8272, 7944, 7947] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97879 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[0, 1, 1, 32759, 0, 32767, 0, 8] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[8275, 8274, 8275, 8273, 8276, 8272, 7944, 7947] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97879 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[35, 0, 0, 32732, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[0, 1, 1, 32759, 0, 32767, 0, 8] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[0, 1, 1, 32759, 0, 32767, 0, 8] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[8275, 8274, 8275, 8273, 8276, 8272, 7944, 7947] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97879 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[0, 1, 1, 32759, 0, 32767, 0, 8] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[35, 0, 0, 32732, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[35, 0, 0, 32732, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[35, 0, 0, 32732, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[7947, 8276, 8275, 8275, 8272, 7944, 8274, 8273] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97904 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[8275, 8274, 8275, 8273, 8276, 8272, 7944, 7947] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97879 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[8275, 8274, 8275, 8273, 8276, 8272, 7944, 7947] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97879 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[8275, 8274, 8275, 8273, 8276, 8272, 7944, 7947] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97879 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[7947, 8276, 8275, 8275, 8272, 7944, 8274, 8273] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97904 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[7947, 8276, 8275, 8275, 8272, 7944, 8274, 8273] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97904 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[7947, 8276, 8275, 8275, 8272, 7944, 8274, 8273] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97904 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[35, 0, 0, 32732, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[32767, 0, 0, 0, 1, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[35, 0, 0, 32732, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[35, 0, 0, 32732, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[32767, 0, 0, 0, 1, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[7947, 8276, 8275, 8275, 8272, 7944, 8274, 8273] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97904 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[35, 0, 0, 32732, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[32767, 0, 0, 0, 1, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[8275, 8276, 8272, 7944, 8273, 8275, 7946, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97968 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[32767, 0, 0, 0, 1, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[7947, 8276, 8275, 8275, 8272, 7944, 8274, 8273] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97904 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[7947, 8276, 8275, 8275, 8272, 7944, 8274, 8273] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97904 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[8275, 8276, 8272, 7944, 8273, 8275, 7946, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97968 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[8275, 8276, 8272, 7944, 8273, 8275, 7946, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97968 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[7947, 8276, 8275, 8275, 8272, 7944, 8274, 8273] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97904 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[8275, 8276, 8272, 7944, 8273, 8275, 7946, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97968 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[0, 2, 0, 32765, 1, 1, 0, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[32767, 0, 0, 0, 1, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[32767, 0, 0, 0, 1, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[0, 2, 0, 32765, 1, 1, 0, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[32767, 0, 0, 0, 1, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[8275, 7946, 8276, 8275, 8273, 8275, 7944, 8272] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[8275, 8276, 8272, 7944, 8273, 8275, 7946, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97968 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[0, 2, 0, 32765, 1, 1, 0, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[0, 2, 0, 32765, 1, 1, 0, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[32767, 0, 0, 0, 1, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[8275, 8276, 8272, 7944, 8273, 8275, 7946, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97968 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[8275, 7946, 8276, 8275, 8273, 8275, 7944, 8272] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[8275, 8276, 8272, 7944, 8273, 8275, 7946, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97968 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[8275, 7946, 8276, 8275, 8273, 8275, 7944, 8272] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[8275, 7946, 8276, 8275, 8273, 8275, 7944, 8272] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[8275, 8276, 8272, 7944, 8273, 8275, 7946, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97968 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[32765, 0, 1, 1, 2, 32767, 0, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[0, 2, 0, 32765, 1, 1, 0, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[0, 2, 0, 32765, 1, 1, 0, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[32765, 0, 1, 1, 2, 32767, 0, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[0, 2, 0, 32765, 1, 1, 0, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[8275, 8276, 8275, 8273, 7946, 8275, 8272, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[32765, 0, 1, 1, 2, 32767, 0, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[8275, 7946, 8276, 8275, 8273, 8275, 7944, 8272] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[32765, 0, 1, 1, 2, 32767, 0, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[0, 2, 0, 32765, 1, 1, 0, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[8275, 7946, 8276, 8275, 8273, 8275, 7944, 8272] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[8275, 8276, 8275, 8273, 7946, 8275, 8272, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[8275, 7946, 8276, 8275, 8273, 8275, 7944, 8272] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[8275, 8276, 8275, 8273, 7946, 8275, 8272, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[1, 32767, 32767, 1, 0, 0, 0, 0] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=4 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[8275, 8276, 8275, 8273, 7946, 8275, 8272, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[8275, 7946, 8276, 8275, 8273, 8275, 7944, 8272] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[32765, 0, 1, 1, 2, 32767, 0, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[1, 32767, 32767, 1, 0, 0, 0, 0] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=4 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[32765, 0, 1, 1, 2, 32767, 0, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[32765, 0, 1, 1, 2, 32767, 0, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[8276, 8276, 8276, 7945, 8275, 8272, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[1, 32767, 32767, 1, 0, 0, 0, 0] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=4 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[8275, 8276, 8275, 8273, 7946, 8275, 8272, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[1, 32767, 32767, 1, 0, 0, 0, 0] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=4 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[32765, 0, 1, 1, 2, 32767, 0, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[8276, 8276, 8276, 7945, 8275, 8272, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[8275, 8276, 8275, 8273, 7946, 8275, 8272, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[8275, 8276, 8275, 8273, 7946, 8275, 8272, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[8276, 8276, 8276, 7945, 8275, 8272, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[8276, 8276, 8276, 7945, 8275, 8272, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[8275, 8276, 8275, 8273, 7946, 8275, 8272, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97961 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[32759, 1, 1, 0, 0, 5, 3, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[1, 32767, 32767, 1, 0, 0, 0, 0] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=4 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[32759, 1, 1, 0, 0, 5, 3, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[1, 32767, 32767, 1, 0, 0, 0, 0] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=4 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[1, 32767, 32767, 1, 0, 0, 0, 0] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=4 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[32759, 1, 1, 0, 0, 5, 3, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[8273, 7945, 8275, 8276, 8275, 7947, 8273, 8272] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[32759, 1, 1, 0, 0, 5, 3, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[8273, 7945, 8275, 8276, 8275, 7947, 8273, 8272] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[8276, 8276, 8276, 7945, 8275, 8272, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[1, 32767, 32767, 1, 0, 0, 0, 0] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=4 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[8276, 8276, 8276, 7945, 8275, 8272, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[8273, 7945, 8275, 8276, 8275, 7947, 8273, 8272] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[8276, 8276, 8276, 7945, 8275, 8272, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[8273, 7945, 8275, 8276, 8275, 7947, 8273, 8272] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[29860, 0, 32767, 1, 2327, 580, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[8276, 8276, 8276, 7945, 8275, 8272, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[29860, 0, 32767, 1, 2327, 580, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[32759, 1, 1, 0, 0, 5, 3, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[32759, 1, 1, 0, 0, 5, 3, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[29860, 0, 32767, 1, 2327, 580, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[32759, 1, 1, 0, 0, 5, 3, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[8274, 8276, 8275, 7945, 7947, 8273, 8272, 8274] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=92240 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[8274, 8276, 8275, 7945, 7947, 8273, 8272, 8274] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=92240 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[29860, 0, 32767, 1, 2327, 580, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[8273, 7945, 8275, 8276, 8275, 7947, 8273, 8272] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[32759, 1, 1, 0, 0, 5, 3, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[8273, 7945, 8275, 8276, 8275, 7947, 8273, 8272] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[8274, 8276, 8275, 7945, 7947, 8273, 8272, 8274] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=92240 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[8273, 7945, 8275, 8276, 8275, 7947, 8273, 8272] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[32767, 0, 1, 4, 13, 0, 32744, 7] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[8274, 8276, 8275, 7945, 7947, 8273, 8272, 8274] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=92240 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[8273, 7945, 8275, 8276, 8275, 7947, 8273, 8272] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[32767, 0, 1, 4, 13, 0, 32744, 7] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[29860, 0, 32767, 1, 2327, 580, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[29860, 0, 32767, 1, 2327, 580, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[32767, 0, 1, 4, 13, 0, 32744, 7] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[8275, 8276, 7945, 8275, 7946, 8272, 8274, 8273] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97805 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[29860, 0, 32767, 1, 2327, 580, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[8275, 8276, 7945, 8275, 7946, 8272, 8274, 8273] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97805 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[8274, 8276, 8275, 7945, 7947, 8273, 8272, 8274] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=92240 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[32767, 0, 1, 4, 13, 0, 32744, 7] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[29860, 0, 32767, 1, 2327, 580, 0, 1] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[8275, 8276, 7945, 8275, 7946, 8272, 8274, 8273] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97805 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[8274, 8276, 8275, 7945, 7947, 8273, 8272, 8274] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=92240 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[8274, 8276, 8275, 7945, 7947, 8273, 8272, 8274] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=92240 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 1, 0, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 1, 0, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[8275, 8276, 7945, 8275, 7946, 8272, 8274, 8273] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97805 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[8274, 8276, 8275, 7945, 7947, 8273, 8272, 8274] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=92240 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[32767, 0, 1, 4, 13, 0, 32744, 7] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 1, 0, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[8276, 8276, 8275, 8276, 8272, 7945, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[32767, 0, 1, 4, 13, 0, 32744, 7] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[32767, 0, 1, 4, 13, 0, 32744, 7] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[8276, 8276, 8275, 8276, 8272, 7945, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[8275, 8276, 7945, 8275, 7946, 8272, 8274, 8273] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97805 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[8276, 8276, 8275, 8276, 8272, 7945, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 1, 0, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[32767, 0, 1, 4, 13, 0, 32744, 7] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[8275, 8276, 7945, 8275, 7946, 8272, 8274, 8273] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97805 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[8275, 8276, 7945, 8275, 7946, 8272, 8274, 8273] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97805 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[1, 1, 0, 0, 32767, 0, 1, 32766] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[1, 1, 0, 0, 32767, 0, 1, 32766] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[8276, 8276, 8275, 8276, 8272, 7945, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[8275, 8276, 7945, 8275, 7946, 8272, 8274, 8273] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97805 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 1, 0, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[1, 1, 0, 0, 32767, 0, 1, 32766] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[8275, 8273, 8276, 8275, 8272, 7944, 7946, 8275] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[8275, 8273, 8276, 8275, 8272, 7944, 7946, 8275] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 1, 0, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 1, 0, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[8276, 8276, 8275, 8276, 8272, 7945, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[8275, 8273, 8276, 8275, 8272, 7944, 7946, 8275] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[1, 1, 0, 0, 32767, 0, 1, 32766] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[1, 0, 32767, 32767, 0, 1, 0, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[8276, 8276, 8275, 8276, 8272, 7945, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[8276, 8276, 8275, 8276, 8272, 7945, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[8275, 8273, 8276, 8275, 8272, 7944, 7946, 8275] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[8276, 8276, 8275, 8276, 8272, 7945, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[1, 1, 0, 0, 32767, 0, 1, 32766] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[1, 1, 0, 0, 32767, 0, 1, 32766] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[1, 1, 0, 0, 32767, 0, 1, 32766] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[8275, 8273, 8276, 8275, 8272, 7944, 7946, 8275] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[1, 1, 0, 0, 32767, 0, 1, 32766] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[8275, 8273, 8276, 8275, 8272, 7944, 7946, 8275] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[8275, 8273, 8276, 8275, 8272, 7944, 7946, 8275] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[8275, 8273, 8276, 8275, 8272, 7944, 7946, 8275] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=32 total_expected_recv=259282 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=64640 peak_demand=10109 required_capacity=1.234009 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=32 total_expected_recv=250612 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=72960 peak_demand=8631 required_capacity=1.053589 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=32 total_expected_recv=261295 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=62208 peak_demand=10107 required_capacity=1.233765 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=32 total_expected_recv=267170 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57344 peak_demand=10106 required_capacity=1.233643 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=32 total_expected_recv=266213 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57984 peak_demand=10106 required_capacity=1.233643 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=32 total_expected_recv=266817 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57984 peak_demand=10107 required_capacity=1.233765 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=32 total_expected_recv=267501 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57216 peak_demand=9259 required_capacity=1.130249 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=32 total_expected_recv=258262 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=65792 peak_demand=9260 required_capacity=1.130371 + [2026-08-30 05:08:05] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 893.9 | throughput per GPU (TFLOP/s/GPU): 373.0 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.399203E+00 | loss scale: 1.0 | grad norm: 5.113 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 10240 | min safe capacity: 1.2500 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=10240 expected_recv=17595 expected_overflow=0 exact_recv_per_rank=[8803, 8801, 7336, 8802, 5468, 8802, 8802, 8722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=10240 expected_recv=17597 expected_overflow=0 exact_recv_per_rank=[8803, 8801, 7336, 8802, 5468, 8802, 8802, 8722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=10240 expected_recv=17595 expected_overflow=0 exact_recv_per_rank=[8803, 8801, 7336, 8802, 5468, 8802, 8802, 8722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=10240 expected_recv=13758 expected_overflow=0 exact_recv_per_rank=[8803, 8801, 7336, 8802, 5468, 8802, 8802, 8722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=10240 expected_recv=17183 expected_overflow=0 exact_recv_per_rank=[8803, 8801, 7336, 8802, 5468, 8802, 8802, 8722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=10240 expected_recv=17516 expected_overflow=0 exact_recv_per_rank=[8803, 8801, 7336, 8802, 5468, 8802, 8802, 8722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=10240 expected_recv=17596 expected_overflow=0 exact_recv_per_rank=[8803, 8801, 7336, 8802, 5468, 8802, 8802, 8722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=10240 expected_recv=17595 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=10240 expected_recv=17597 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=10240 expected_recv=17595 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=10240 expected_recv=12232 expected_overflow=0 exact_recv_per_rank=[8803, 8801, 7336, 8802, 5468, 8802, 8802, 8722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=10240 expected_recv=13758 actual_valid_recv=13952 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=10240 expected_recv=17183 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17664 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17664 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=10240 expected_recv=17516 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=10240 expected_recv=17596 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17920 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=13952 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17536 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=10240 expected_recv=12232 actual_valid_recv=12416 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17920 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=17664 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=2 received_num_tokens=20480 valid_num_tokens=12416 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8276, 8274, 7945, 7946, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8276, 8274, 7945, 7946, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8276, 8274, 7945, 7946, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8276, 8274, 7945, 7946, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8276, 8274, 7945, 7946, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8276, 8274, 7945, 7946, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=10240 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8276, 8274, 7945, 7946, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=10240 expected_recv=15895 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8276, 8274, 7945, 7946, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=10240 expected_recv=15888 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=10240 expected_recv=15895 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=10240 expected_recv=17204 expected_overflow=0 exact_recv_per_rank=[6557, 5973, 8929, 8930, 8931, 8931, 8354, 8931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=10240 expected_recv=17202 expected_overflow=0 exact_recv_per_rank=[6557, 5973, 8929, 8930, 8931, 8931, 8354, 8931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=10240 expected_recv=14246 expected_overflow=0 exact_recv_per_rank=[6557, 5973, 8929, 8930, 8931, 8931, 8354, 8931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=10240 expected_recv=17204 expected_overflow=0 exact_recv_per_rank=[6557, 5973, 8929, 8930, 8931, 8931, 8354, 8931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=10240 expected_recv=16295 expected_overflow=0 exact_recv_per_rank=[6557, 5973, 8929, 8930, 8931, 8931, 8354, 8931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=10240 expected_recv=17204 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=10240 expected_recv=14508 expected_overflow=0 exact_recv_per_rank=[6557, 5973, 8929, 8930, 8931, 8931, 8354, 8931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=10240 expected_recv=17207 expected_overflow=0 exact_recv_per_rank=[6557, 5973, 8929, 8930, 8931, 8931, 8354, 8931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=10240 expected_recv=17206 expected_overflow=0 exact_recv_per_rank=[6557, 5973, 8929, 8930, 8931, 8931, 8354, 8931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=10240 expected_recv=17202 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=10240 expected_recv=14246 actual_valid_recv=14592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=10240 expected_recv=17204 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=17280 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=10240 expected_recv=14508 actual_valid_recv=14976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=10240 expected_recv=16295 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=10240 expected_recv=17207 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=10240 expected_recv=17206 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=17280 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=14592 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=14976 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=17280 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=2 received_num_tokens=20480 valid_num_tokens=17280 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=10240 expected_recv=17183 expected_overflow=0 exact_recv_per_rank=[5949, 8593, 8592, 8592, 8591, 8592, 8035, 8592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=10240 expected_recv=17182 expected_overflow=0 exact_recv_per_rank=[5949, 8593, 8592, 8592, 8591, 8592, 8035, 8592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=10240 expected_recv=17183 expected_overflow=0 exact_recv_per_rank=[5949, 8593, 8592, 8592, 8591, 8592, 8035, 8592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=10240 expected_recv=14491 expected_overflow=0 exact_recv_per_rank=[5949, 8593, 8592, 8592, 8591, 8592, 8035, 8592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=10240 expected_recv=17183 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=10240 expected_recv=17182 expected_overflow=0 exact_recv_per_rank=[5949, 8593, 8592, 8592, 8591, 8592, 8035, 8592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=10240 expected_recv=16129 expected_overflow=0 exact_recv_per_rank=[5949, 8593, 8592, 8592, 8591, 8592, 8035, 8592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=10240 expected_recv=14540 expected_overflow=0 exact_recv_per_rank=[5949, 8593, 8592, 8592, 8591, 8592, 8035, 8592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=10240 expected_recv=17182 expected_overflow=0 exact_recv_per_rank=[5949, 8593, 8592, 8592, 8591, 8592, 8035, 8592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=10240 expected_recv=17182 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=10240 expected_recv=17183 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=10240 expected_recv=14491 actual_valid_recv=14848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=10240 expected_recv=16129 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=10240 expected_recv=17182 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=10240 expected_recv=14540 actual_valid_recv=14976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17664 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=10240 expected_recv=17182 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17664 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=14848 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17664 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=14976 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=2 received_num_tokens=20480 valid_num_tokens=17408 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8273, 8273, 7945, 8273, 8275, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8273, 8273, 7945, 8273, 8275, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=10240 expected_recv=16224 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8273, 8273, 7945, 8273, 8275, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=10240 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8273, 8273, 7945, 8273, 8275, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=10240 expected_recv=16224 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8273, 8273, 7945, 8273, 8275, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=10240 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8273, 8273, 7945, 8273, 8275, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8273, 8273, 7945, 8273, 8275, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8273, 8273, 7945, 8273, 8275, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=10240 expected_recv=16224 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=10240 expected_recv=16216 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=10240 expected_recv=16224 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=10240 expected_recv=16220 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=10240 expected_recv=15893 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8273, 7947, 8274, 8275, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=10240 expected_recv=15893 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8273, 7947, 8274, 8275, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8273, 7947, 8274, 8275, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=10240 expected_recv=16221 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8273, 7947, 8274, 8275, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=10240 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8273, 7947, 8274, 8275, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8273, 7947, 8274, 8275, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8273, 7947, 8274, 8275, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8273, 7947, 8274, 8275, 8272, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=10240 expected_recv=16221 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=10240 expected_recv=16217 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8273, 8274, 8274, 7945, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8273, 8274, 8274, 7945, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8273, 8274, 8274, 7945, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=10240 expected_recv=15890 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8273, 8274, 8274, 7945, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8273, 8274, 8274, 7945, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8273, 8274, 8274, 7945, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8273, 8274, 8274, 7945, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=10240 expected_recv=15890 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=10240 expected_recv=15891 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8273, 8274, 8274, 7945, 7946, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 7943, 8275, 8273, 7950, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=10240 expected_recv=15891 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 7943, 8275, 8273, 7950, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=10240 expected_recv=16209 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 7943, 8275, 8273, 7950, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=10240 expected_recv=16224 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 7943, 8275, 8273, 7950, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=10240 expected_recv=16209 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 7943, 8275, 8273, 7950, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=10240 expected_recv=16230 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 7943, 8275, 8273, 7950, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=10240 expected_recv=16224 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 7943, 8275, 8273, 7950, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=10240 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 7943, 8275, 8273, 7950, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7945, 7949, 8272, 8274, 8275, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=10240 expected_recv=16230 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=10240 expected_recv=16545 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[7945, 7949, 8272, 8274, 8275, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[7945, 7949, 8272, 8274, 8275, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7945, 7949, 8272, 8274, 8275, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[7945, 7949, 8272, 8274, 8275, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=10240 expected_recv=16230 expected_overflow=0 exact_recv_per_rank=[7945, 7949, 8272, 8274, 8275, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=10240 expected_recv=16223 expected_overflow=0 exact_recv_per_rank=[7945, 7949, 8272, 8274, 8275, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=10240 expected_recv=16210 expected_overflow=0 exact_recv_per_rank=[7945, 7949, 8272, 8274, 8275, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=10240 expected_recv=9408 expected_overflow=0 exact_recv_per_rank=[11035, 6, 11032, 4, 10695, 11034, 11035, 10695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=10240 expected_recv=16230 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=10240 expected_recv=16223 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=10240 expected_recv=16210 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=10240 expected_recv=9408 actual_valid_recv=9728 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=10240 expected_recv=20127 expected_overflow=455 exact_recv_per_rank=[11035, 6, 11032, 4, 10695, 11034, 11035, 10695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=9728 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=10240 expected_recv=11043 expected_overflow=792 exact_recv_per_rank=[11035, 6, 11032, 4, 10695, 11034, 11035, 10695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=10240 expected_recv=20127 actual_valid_recv=19712 observed_loss=455 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=10240 expected_recv=11043 actual_valid_recv=10368 observed_loss=792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=19712 slack_slots=768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=10240 expected_recv=22697 expected_overflow=2217 exact_recv_per_rank=[11035, 6, 11032, 4, 10695, 11034, 11035, 10695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=10368 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=10240 expected_recv=22697 expected_overflow=2217 exact_recv_per_rank=[11035, 6, 11032, 4, 10695, 11034, 11035, 10695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=10240 expected_recv=22358 expected_overflow=1878 exact_recv_per_rank=[11035, 6, 11032, 4, 10695, 11034, 11035, 10695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=10240 expected_recv=45 expected_overflow=0 exact_recv_per_rank=[11035, 6, 11032, 4, 10695, 11034, 11035, 10695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=10240 expected_recv=22697 actual_valid_recv=20480 observed_loss=2217 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=10240 expected_recv=16222 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 8274, 7944, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=10240 expected_recv=22697 expected_overflow=2217 exact_recv_per_rank=[11035, 6, 11032, 4, 10695, 11034, 11035, 10695] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=10240 expected_recv=22697 actual_valid_recv=20480 observed_loss=2217 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=10240 expected_recv=45 actual_valid_recv=256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=10240 expected_recv=22358 actual_valid_recv=20480 observed_loss=1878 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=20480 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=10240 expected_recv=16222 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=20480 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=10240 expected_recv=22697 actual_valid_recv=20480 observed_loss=2217 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=256 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=20480 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=2 received_num_tokens=20480 valid_num_tokens=20480 slack_slots=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=10240 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 8274, 7944, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=10240 expected_recv=16209 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 8274, 7944, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=10240 expected_recv=16545 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=10240 expected_recv=16209 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 8274, 7944, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 8274, 7944, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 8274, 7944, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 8274, 8272, 7950, 7943, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 8274, 7944, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=10240 expected_recv=15901 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 8274, 7944, 8273, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=10240 expected_recv=15901 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 8274, 8272, 7950, 7943, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 8274, 8272, 7950, 7943, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=10240 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 8274, 8272, 7950, 7943, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 8274, 8272, 7950, 7943, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=10240 expected_recv=15886 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 8273, 7947, 8273, 7946, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=10240 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 8274, 8272, 7950, 7943, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=10240 expected_recv=16217 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=10240 expected_recv=16206 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 8274, 8272, 7950, 7943, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=10240 expected_recv=15886 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=10240 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=10240 expected_recv=16206 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=10240 expected_recv=15907 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 8274, 8272, 7950, 7943, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 8273, 7947, 8273, 7946, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=10240 expected_recv=15907 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 8273, 7947, 8273, 7946, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 8273, 7947, 8273, 7946, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 8273, 7947, 8273, 7946, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=10240 expected_recv=15906 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7949, 7941, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 8273, 7947, 8273, 7946, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 8273, 7947, 8273, 7946, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=10240 expected_recv=15906 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=10240 expected_recv=15898 expected_overflow=0 exact_recv_per_rank=[8272, 8275, 8273, 7947, 8273, 7946, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7949, 7941, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16128 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=10240 expected_recv=15898 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7949, 7941, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=12 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=10240 expected_recv=16208 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7949, 7941, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7949, 7941, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=10240 expected_recv=16208 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 7944, 8273, 8272, 8275, 8274, 7949, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=10240 expected_recv=16215 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7949, 7941, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16384 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7949, 7941, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=10240 expected_recv=16215 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7949, 7941, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=10240 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 7944, 8273, 8272, 8275, 8274, 7949, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 7944, 8273, 8272, 8275, 8274, 7949, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=10240 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=10240 expected_recv=15887 expected_overflow=0 exact_recv_per_rank=[8275, 7944, 8273, 8272, 8275, 8274, 7949, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=10240 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8275, 7944, 8273, 8272, 8275, 8274, 7949, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=10240 expected_recv=15887 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=10240 expected_recv=16224 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8272, 7950, 8275, 8275, 7941, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=10240 expected_recv=16231 expected_overflow=0 exact_recv_per_rank=[8275, 7944, 8273, 8272, 8275, 8274, 7949, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16256 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=10240 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=10240 expected_recv=16224 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=10240 expected_recv=16231 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=10240 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8275, 7944, 8273, 8272, 8275, 8274, 7949, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=10240 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 7944, 8273, 8272, 8275, 8274, 7949, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8272, 7950, 8275, 8275, 7941, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=10240 expected_recv=16545 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=10240 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=10240 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8272, 7950, 8275, 8275, 7941, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=2 received_num_tokens=20480 valid_num_tokens=16768 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=10240 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=10240 expected_recv=16215 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8272, 7950, 8275, 8275, 7941, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=10240 expected_recv=16215 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=10240 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8272, 7950, 8275, 8275, 7941, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=10240 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8272, 7950, 8275, 8275, 7941, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=10240 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[18905, 77, 5037, 11101, 4090, 26313, 4, 9] avg=8192.0000 max=26313 max_rank=5 min=4 min_rank=6 imbalance_ratio=3.212036 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=10240 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16896 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=10240 expected_recv=16209 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8272, 7950, 8275, 8275, 7941, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=10240 expected_recv=16232 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8272, 7950, 8275, 8275, 7941, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[18905, 77, 5037, 11101, 4090, 26313, 4, 9] avg=8192.0000 max=26313 max_rank=5 min=4 min_rank=6 imbalance_ratio=3.212036 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=10240 expected_recv=16209 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=10240 expected_recv=16232 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[18905, 77, 5037, 11101, 4090, 26313, 4, 9] avg=8192.0000 max=26313 max_rank=5 min=4 min_rank=6 imbalance_ratio=3.212036 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16512 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[8803, 8801, 7336, 8802, 5468, 8802, 8802, 8722] avg=8192.0000 max=8803 max_rank=0 min=5468 min_rank=4 imbalance_ratio=1.074585 max_load_reduction=17510 max_load_reduction_pct=0.665451 moved_to_replicas=57721 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=2 received_num_tokens=20480 valid_num_tokens=16640 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[18905, 77, 5037, 11101, 4090, 26313, 4, 9] avg=8192.0000 max=26313 max_rank=5 min=4 min_rank=6 imbalance_ratio=3.212036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[8803, 8801, 7336, 8802, 5468, 8802, 8802, 8722] avg=8192.0000 max=8803 max_rank=0 min=5468 min_rank=4 imbalance_ratio=1.074585 max_load_reduction=17510 max_load_reduction_pct=0.665451 moved_to_replicas=57721 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[18905, 77, 5037, 11101, 4090, 26313, 4, 9] avg=8192.0000 max=26313 max_rank=5 min=4 min_rank=6 imbalance_ratio=3.212036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[8803, 8801, 7336, 8802, 5468, 8802, 8802, 8722] avg=8192.0000 max=8803 max_rank=0 min=5468 min_rank=4 imbalance_ratio=1.074585 max_load_reduction=17510 max_load_reduction_pct=0.665451 moved_to_replicas=57721 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[18905, 77, 5037, 11101, 4090, 26313, 4, 9] avg=8192.0000 max=26313 max_rank=5 min=4 min_rank=6 imbalance_ratio=3.212036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[8803, 8801, 7336, 8802, 5468, 8802, 8802, 8722] avg=8192.0000 max=8803 max_rank=0 min=5468 min_rank=4 imbalance_ratio=1.074585 max_load_reduction=17510 max_load_reduction_pct=0.665451 moved_to_replicas=57721 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[2, 32544, 32763, 4, 174, 3, 45, 1] avg=8192.0000 max=32763 max_rank=2 min=1 min_rank=7 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[18905, 77, 5037, 11101, 4090, 26313, 4, 9] avg=8192.0000 max=26313 max_rank=5 min=4 min_rank=6 imbalance_ratio=3.212036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[18905, 77, 5037, 11101, 4090, 26313, 4, 9] avg=8192.0000 max=26313 max_rank=5 min=4 min_rank=6 imbalance_ratio=3.212036 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[8803, 8801, 7336, 8802, 5468, 8802, 8802, 8722] avg=8192.0000 max=8803 max_rank=0 min=5468 min_rank=4 imbalance_ratio=1.074585 max_load_reduction=17510 max_load_reduction_pct=0.665451 moved_to_replicas=57721 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[2, 32544, 32763, 4, 174, 3, 45, 1] avg=8192.0000 max=32763 max_rank=2 min=1 min_rank=7 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[8803, 8801, 7336, 8802, 5468, 8802, 8802, 8722] avg=8192.0000 max=8803 max_rank=0 min=5468 min_rank=4 imbalance_ratio=1.074585 max_load_reduction=17510 max_load_reduction_pct=0.665451 moved_to_replicas=57721 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[2, 32544, 32763, 4, 174, 3, 45, 1] avg=8192.0000 max=32763 max_rank=2 min=1 min_rank=7 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[8273, 8274, 8276, 8274, 7945, 7946, 8274, 8274] avg=8192.0000 max=8276 max_rank=2 min=7945 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97222 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[8803, 8801, 7336, 8802, 5468, 8802, 8802, 8722] avg=8192.0000 max=8803 max_rank=0 min=5468 min_rank=4 imbalance_ratio=1.074585 max_load_reduction=17510 max_load_reduction_pct=0.665451 moved_to_replicas=57721 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[8803, 8801, 7336, 8802, 5468, 8802, 8802, 8722] avg=8192.0000 max=8803 max_rank=0 min=5468 min_rank=4 imbalance_ratio=1.074585 max_load_reduction=17510 max_load_reduction_pct=0.665451 moved_to_replicas=57721 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[2, 32544, 32763, 4, 174, 3, 45, 1] avg=8192.0000 max=32763 max_rank=2 min=1 min_rank=7 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[8273, 8274, 8276, 8274, 7945, 7946, 8274, 8274] avg=8192.0000 max=8276 max_rank=2 min=7945 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97222 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[2, 32544, 32763, 4, 174, 3, 45, 1] avg=8192.0000 max=32763 max_rank=2 min=1 min_rank=7 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[8273, 8274, 8276, 8274, 7945, 7946, 8274, 8274] avg=8192.0000 max=8276 max_rank=2 min=7945 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97222 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[2, 32544, 32763, 4, 174, 3, 45, 1] avg=8192.0000 max=32763 max_rank=2 min=1 min_rank=7 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[5, 2, 32762, 15482, 17281, 0, 4, 0] avg=8192.0000 max=32762 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[8273, 8274, 8276, 8274, 7945, 7946, 8274, 8274] avg=8192.0000 max=8276 max_rank=2 min=7945 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97222 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[2, 32544, 32763, 4, 174, 3, 45, 1] avg=8192.0000 max=32763 max_rank=2 min=1 min_rank=7 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[8273, 8274, 8276, 8274, 7945, 7946, 8274, 8274] avg=8192.0000 max=8276 max_rank=2 min=7945 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97222 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[5, 2, 32762, 15482, 17281, 0, 4, 0] avg=8192.0000 max=32762 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[2, 32544, 32763, 4, 174, 3, 45, 1] avg=8192.0000 max=32763 max_rank=2 min=1 min_rank=7 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[8273, 8274, 8276, 8274, 7945, 7946, 8274, 8274] avg=8192.0000 max=8276 max_rank=2 min=7945 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97222 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[6557, 5973, 8929, 8930, 8931, 8931, 8354, 8931] avg=8192.0000 max=8931 max_rank=4 min=5973 min_rank=1 imbalance_ratio=1.090210 max_load_reduction=23831 max_load_reduction_pct=0.727398 moved_to_replicas=79430 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[5, 2, 32762, 15482, 17281, 0, 4, 0] avg=8192.0000 max=32762 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[5, 2, 32762, 15482, 17281, 0, 4, 0] avg=8192.0000 max=32762 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[8273, 8274, 8276, 8274, 7945, 7946, 8274, 8274] avg=8192.0000 max=8276 max_rank=2 min=7945 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97222 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[6557, 5973, 8929, 8930, 8931, 8931, 8354, 8931] avg=8192.0000 max=8931 max_rank=4 min=5973 min_rank=1 imbalance_ratio=1.090210 max_load_reduction=23831 max_load_reduction_pct=0.727398 moved_to_replicas=79430 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[8273, 8274, 8276, 8274, 7945, 7946, 8274, 8274] avg=8192.0000 max=8276 max_rank=2 min=7945 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97222 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[5, 2, 32762, 15482, 17281, 0, 4, 0] avg=8192.0000 max=32762 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[6557, 5973, 8929, 8930, 8931, 8931, 8354, 8931] avg=8192.0000 max=8931 max_rank=4 min=5973 min_rank=1 imbalance_ratio=1.090210 max_load_reduction=23831 max_load_reduction_pct=0.727398 moved_to_replicas=79430 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[5, 2, 32762, 15482, 17281, 0, 4, 0] avg=8192.0000 max=32762 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[2, 0, 3, 31588, 1, 31722, 2217, 3] avg=8192.0000 max=31722 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.872314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[6557, 5973, 8929, 8930, 8931, 8931, 8354, 8931] avg=8192.0000 max=8931 max_rank=4 min=5973 min_rank=1 imbalance_ratio=1.090210 max_load_reduction=23831 max_load_reduction_pct=0.727398 moved_to_replicas=79430 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[6557, 5973, 8929, 8930, 8931, 8931, 8354, 8931] avg=8192.0000 max=8931 max_rank=4 min=5973 min_rank=1 imbalance_ratio=1.090210 max_load_reduction=23831 max_load_reduction_pct=0.727398 moved_to_replicas=79430 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[6557, 5973, 8929, 8930, 8931, 8931, 8354, 8931] avg=8192.0000 max=8931 max_rank=4 min=5973 min_rank=1 imbalance_ratio=1.090210 max_load_reduction=23831 max_load_reduction_pct=0.727398 moved_to_replicas=79430 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[2, 0, 3, 31588, 1, 31722, 2217, 3] avg=8192.0000 max=31722 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.872314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[5, 2, 32762, 15482, 17281, 0, 4, 0] avg=8192.0000 max=32762 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[5949, 8593, 8592, 8592, 8591, 8592, 8035, 8592] avg=8192.0000 max=8593 max_rank=1 min=5949 min_rank=0 imbalance_ratio=1.048950 max_load_reduction=23129 max_load_reduction_pct=0.729115 moved_to_replicas=92093 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[5, 2, 32762, 15482, 17281, 0, 4, 0] avg=8192.0000 max=32762 max_rank=2 min=0 min_rank=5 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[2, 0, 3, 31588, 1, 31722, 2217, 3] avg=8192.0000 max=31722 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.872314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[2, 0, 3, 31588, 1, 31722, 2217, 3] avg=8192.0000 max=31722 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.872314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[5949, 8593, 8592, 8592, 8591, 8592, 8035, 8592] avg=8192.0000 max=8593 max_rank=1 min=5949 min_rank=0 imbalance_ratio=1.048950 max_load_reduction=23129 max_load_reduction_pct=0.729115 moved_to_replicas=92093 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[6557, 5973, 8929, 8930, 8931, 8931, 8354, 8931] avg=8192.0000 max=8931 max_rank=4 min=5973 min_rank=1 imbalance_ratio=1.090210 max_load_reduction=23831 max_load_reduction_pct=0.727398 moved_to_replicas=79430 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[6557, 5973, 8929, 8930, 8931, 8931, 8354, 8931] avg=8192.0000 max=8931 max_rank=4 min=5973 min_rank=1 imbalance_ratio=1.090210 max_load_reduction=23831 max_load_reduction_pct=0.727398 moved_to_replicas=79430 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[2, 0, 3, 31588, 1, 31722, 2217, 3] avg=8192.0000 max=31722 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.872314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[2, 0, 3, 31588, 1, 31722, 2217, 3] avg=8192.0000 max=31722 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.872314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[5949, 8593, 8592, 8592, 8591, 8592, 8035, 8592] avg=8192.0000 max=8593 max_rank=1 min=5949 min_rank=0 imbalance_ratio=1.048950 max_load_reduction=23129 max_load_reduction_pct=0.729115 moved_to_replicas=92093 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[3, 3, 32761, 32764, 2, 0, 0, 3] avg=8192.0000 max=32764 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[5949, 8593, 8592, 8592, 8591, 8592, 8035, 8592] avg=8192.0000 max=8593 max_rank=1 min=5949 min_rank=0 imbalance_ratio=1.048950 max_load_reduction=23129 max_load_reduction_pct=0.729115 moved_to_replicas=92093 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[5949, 8593, 8592, 8592, 8591, 8592, 8035, 8592] avg=8192.0000 max=8593 max_rank=1 min=5949 min_rank=0 imbalance_ratio=1.048950 max_load_reduction=23129 max_load_reduction_pct=0.729115 moved_to_replicas=92093 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[3, 3, 32761, 32764, 2, 0, 0, 3] avg=8192.0000 max=32764 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[5949, 8593, 8592, 8592, 8591, 8592, 8035, 8592] avg=8192.0000 max=8593 max_rank=1 min=5949 min_rank=0 imbalance_ratio=1.048950 max_load_reduction=23129 max_load_reduction_pct=0.729115 moved_to_replicas=92093 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[2, 0, 3, 31588, 1, 31722, 2217, 3] avg=8192.0000 max=31722 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.872314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[8274, 8274, 8273, 8273, 7945, 8273, 8275, 7949] avg=8192.0000 max=8275 max_rank=6 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[2, 0, 3, 31588, 1, 31722, 2217, 3] avg=8192.0000 max=31722 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.872314 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[3, 3, 32761, 32764, 2, 0, 0, 3] avg=8192.0000 max=32764 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[3, 3, 32761, 32764, 2, 0, 0, 3] avg=8192.0000 max=32764 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[8274, 8274, 8273, 8273, 7945, 8273, 8275, 7949] avg=8192.0000 max=8275 max_rank=6 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[5949, 8593, 8592, 8592, 8591, 8592, 8035, 8592] avg=8192.0000 max=8593 max_rank=1 min=5949 min_rank=0 imbalance_ratio=1.048950 max_load_reduction=23129 max_load_reduction_pct=0.729115 moved_to_replicas=92093 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[5949, 8593, 8592, 8592, 8591, 8592, 8035, 8592] avg=8192.0000 max=8593 max_rank=1 min=5949 min_rank=0 imbalance_ratio=1.048950 max_load_reduction=23129 max_load_reduction_pct=0.729115 moved_to_replicas=92093 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[3, 3, 32761, 32764, 2, 0, 0, 3] avg=8192.0000 max=32764 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[8274, 8274, 8273, 8273, 7945, 8273, 8275, 7949] avg=8192.0000 max=8275 max_rank=6 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[3, 3, 32761, 32764, 2, 0, 0, 3] avg=8192.0000 max=32764 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[0, 3, 4, 1229, 126, 32764, 1, 31409] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[8274, 8274, 8273, 8273, 7945, 8273, 8275, 7949] avg=8192.0000 max=8275 max_rank=6 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[8274, 8274, 8273, 8273, 7945, 8273, 8275, 7949] avg=8192.0000 max=8275 max_rank=6 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[0, 3, 4, 1229, 126, 32764, 1, 31409] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[8274, 8274, 8273, 8273, 7945, 8273, 8275, 7949] avg=8192.0000 max=8275 max_rank=6 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[3, 3, 32761, 32764, 2, 0, 0, 3] avg=8192.0000 max=32764 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[3, 3, 32761, 32764, 2, 0, 0, 3] avg=8192.0000 max=32764 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[8275, 7946, 8273, 7947, 8274, 8275, 8272, 8274] avg=8192.0000 max=8275 max_rank=0 min=7946 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=95270 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[0, 3, 4, 1229, 126, 32764, 1, 31409] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[0, 3, 4, 1229, 126, 32764, 1, 31409] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[8275, 7946, 8273, 7947, 8274, 8275, 8272, 8274] avg=8192.0000 max=8275 max_rank=0 min=7946 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=95270 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[0, 3, 4, 1229, 126, 32764, 1, 31409] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[8274, 8274, 8273, 8273, 7945, 8273, 8275, 7949] avg=8192.0000 max=8275 max_rank=6 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[8274, 8274, 8273, 8273, 7945, 8273, 8275, 7949] avg=8192.0000 max=8275 max_rank=6 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[8275, 7946, 8273, 7947, 8274, 8275, 8272, 8274] avg=8192.0000 max=8275 max_rank=0 min=7946 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=95270 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[0, 3, 4, 1229, 126, 32764, 1, 31409] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[8275, 7946, 8273, 7947, 8274, 8275, 8272, 8274] avg=8192.0000 max=8275 max_rank=0 min=7946 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=95270 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[59, 1, 1, 32549, 32764, 155, 3, 4] avg=8192.0000 max=32764 max_rank=4 min=1 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[8275, 7946, 8273, 7947, 8274, 8275, 8272, 8274] avg=8192.0000 max=8275 max_rank=0 min=7946 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=95270 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[59, 1, 1, 32549, 32764, 155, 3, 4] avg=8192.0000 max=32764 max_rank=4 min=1 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[8275, 7946, 8273, 7947, 8274, 8275, 8272, 8274] avg=8192.0000 max=8275 max_rank=0 min=7946 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=95270 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[0, 3, 4, 1229, 126, 32764, 1, 31409] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[8273, 8276, 8273, 8274, 8274, 7945, 7946, 8275] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97520 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[0, 3, 4, 1229, 126, 32764, 1, 31409] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[59, 1, 1, 32549, 32764, 155, 3, 4] avg=8192.0000 max=32764 max_rank=4 min=1 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[59, 1, 1, 32549, 32764, 155, 3, 4] avg=8192.0000 max=32764 max_rank=4 min=1 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[8273, 8276, 8273, 8274, 8274, 7945, 7946, 8275] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97520 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[59, 1, 1, 32549, 32764, 155, 3, 4] avg=8192.0000 max=32764 max_rank=4 min=1 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[8275, 7946, 8273, 7947, 8274, 8275, 8272, 8274] avg=8192.0000 max=8275 max_rank=0 min=7946 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=95270 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[8275, 7946, 8273, 7947, 8274, 8275, 8272, 8274] avg=8192.0000 max=8275 max_rank=0 min=7946 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=95270 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[8273, 8276, 8273, 8274, 8274, 7945, 7946, 8275] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97520 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[8273, 8276, 8273, 8274, 8274, 7945, 7946, 8275] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97520 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[59, 1, 1, 32549, 32764, 155, 3, 4] avg=8192.0000 max=32764 max_rank=4 min=1 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[32762, 0, 0, 0, 4, 32764, 4, 2] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[8273, 8276, 8273, 8274, 8274, 7945, 7946, 8275] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97520 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[32762, 0, 0, 0, 4, 32764, 4, 2] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[8273, 8276, 8273, 8274, 8274, 7945, 7946, 8275] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97520 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[59, 1, 1, 32549, 32764, 155, 3, 4] avg=8192.0000 max=32764 max_rank=4 min=1 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[8274, 8275, 8273, 7943, 8275, 8273, 7950, 8273] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[59, 1, 1, 32549, 32764, 155, 3, 4] avg=8192.0000 max=32764 max_rank=4 min=1 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[32762, 0, 0, 0, 4, 32764, 4, 2] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[32762, 0, 0, 0, 4, 32764, 4, 2] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[8274, 8275, 8273, 7943, 8275, 8273, 7950, 8273] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[32762, 0, 0, 0, 4, 32764, 4, 2] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[8273, 8276, 8273, 8274, 8274, 7945, 7946, 8275] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97520 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[8273, 8276, 8273, 8274, 8274, 7945, 7946, 8275] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97520 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[32762, 0, 0, 0, 4, 32764, 4, 2] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[8274, 8275, 8273, 7943, 8275, 8273, 7950, 8273] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[8274, 8275, 8273, 7943, 8275, 8273, 7950, 8273] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[2, 6, 0, 32758, 4, 2, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[8274, 8275, 8273, 7943, 8275, 8273, 7950, 8273] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[2, 6, 0, 32758, 4, 2, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[8274, 8275, 8273, 7943, 8275, 8273, 7950, 8273] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[32762, 0, 0, 0, 4, 32764, 4, 2] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[7945, 7949, 8272, 8274, 8275, 8272, 8275, 8274] avg=8192.0000 max=8275 max_rank=4 min=7945 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97929 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[32762, 0, 0, 0, 4, 32764, 4, 2] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[7945, 7949, 8272, 8274, 8275, 8272, 8275, 8274] avg=8192.0000 max=8275 max_rank=4 min=7945 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97929 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[2, 6, 0, 32758, 4, 2, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[2, 6, 0, 32758, 4, 2, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[2, 6, 0, 32758, 4, 2, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[8274, 8275, 8273, 7943, 8275, 8273, 7950, 8273] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[8274, 8275, 8273, 7943, 8275, 8273, 7950, 8273] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[7945, 7949, 8272, 8274, 8275, 8272, 8275, 8274] avg=8192.0000 max=8275 max_rank=4 min=7945 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97929 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[2, 6, 0, 32758, 4, 2, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[22063, 6, 4, 4, 10695, 32764, 0, 0] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[7945, 7949, 8272, 8274, 8275, 8272, 8275, 8274] avg=8192.0000 max=8275 max_rank=4 min=7945 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97929 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[7945, 7949, 8272, 8274, 8275, 8272, 8275, 8274] avg=8192.0000 max=8275 max_rank=4 min=7945 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97929 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[22063, 6, 4, 4, 10695, 32764, 0, 0] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[11035, 6, 11032, 4, 10695, 11034, 11035, 10695] avg=8192.0000 max=11035 max_rank=0 min=4 min_rank=3 imbalance_ratio=1.347046 max_load_reduction=21729 max_load_reduction_pct=0.663197 moved_to_replicas=63246 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[2, 6, 0, 32758, 4, 2, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[7945, 7949, 8272, 8274, 8275, 8272, 8275, 8274] avg=8192.0000 max=8275 max_rank=4 min=7945 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97929 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[2, 6, 0, 32758, 4, 2, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[22063, 6, 4, 4, 10695, 32764, 0, 0] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[11035, 6, 11032, 4, 10695, 11034, 11035, 10695] avg=8192.0000 max=11035 max_rank=0 min=4 min_rank=3 imbalance_ratio=1.347046 max_load_reduction=21729 max_load_reduction_pct=0.663197 moved_to_replicas=63246 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[22063, 6, 4, 4, 10695, 32764, 0, 0] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[22063, 6, 4, 4, 10695, 32764, 0, 0] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[7945, 7949, 8272, 8274, 8275, 8272, 8275, 8274] avg=8192.0000 max=8275 max_rank=4 min=7945 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97929 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[7945, 7949, 8272, 8274, 8275, 8272, 8275, 8274] avg=8192.0000 max=8275 max_rank=4 min=7945 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97929 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[11035, 6, 11032, 4, 10695, 11034, 11035, 10695] avg=8192.0000 max=11035 max_rank=0 min=4 min_rank=3 imbalance_ratio=1.347046 max_load_reduction=21729 max_load_reduction_pct=0.663197 moved_to_replicas=63246 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[5, 32764, 6, 4, 0, 390, 32365, 2] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[22063, 6, 4, 4, 10695, 32764, 0, 0] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[11035, 6, 11032, 4, 10695, 11034, 11035, 10695] avg=8192.0000 max=11035 max_rank=0 min=4 min_rank=3 imbalance_ratio=1.347046 max_load_reduction=21729 max_load_reduction_pct=0.663197 moved_to_replicas=63246 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[11035, 6, 11032, 4, 10695, 11034, 11035, 10695] avg=8192.0000 max=11035 max_rank=0 min=4 min_rank=3 imbalance_ratio=1.347046 max_load_reduction=21729 max_load_reduction_pct=0.663197 moved_to_replicas=63246 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[5, 32764, 6, 4, 0, 390, 32365, 2] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 8274, 7944, 8273, 8273] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=96375 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[22063, 6, 4, 4, 10695, 32764, 0, 0] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[22063, 6, 4, 4, 10695, 32764, 0, 0] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[11035, 6, 11032, 4, 10695, 11034, 11035, 10695] avg=8192.0000 max=11035 max_rank=0 min=4 min_rank=3 imbalance_ratio=1.347046 max_load_reduction=21729 max_load_reduction_pct=0.663197 moved_to_replicas=63246 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 8274, 7944, 8273, 8273] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=96375 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[5, 32764, 6, 4, 0, 390, 32365, 2] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[5, 32764, 6, 4, 0, 390, 32365, 2] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[5, 32764, 6, 4, 0, 390, 32365, 2] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[11035, 6, 11032, 4, 10695, 11034, 11035, 10695] avg=8192.0000 max=11035 max_rank=0 min=4 min_rank=3 imbalance_ratio=1.347046 max_load_reduction=21729 max_load_reduction_pct=0.663197 moved_to_replicas=63246 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[11035, 6, 11032, 4, 10695, 11034, 11035, 10695] avg=8192.0000 max=11035 max_rank=0 min=4 min_rank=3 imbalance_ratio=1.347046 max_load_reduction=21729 max_load_reduction_pct=0.663197 moved_to_replicas=63246 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 8274, 7944, 8273, 8273] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=96375 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[0, 4, 4, 29147, 0, 3617, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 8274, 7944, 8273, 8273] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=96375 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[5, 32764, 6, 4, 0, 390, 32365, 2] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[0, 4, 4, 29147, 0, 3617, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 8274, 7944, 8273, 8273] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=96375 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[8275, 8274, 8274, 8274, 8272, 7950, 7943, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=90082 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[5, 32764, 6, 4, 0, 390, 32365, 2] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[5, 32764, 6, 4, 0, 390, 32365, 2] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 8274, 7944, 8273, 8273] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=96375 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[8275, 8274, 8274, 8274, 8272, 7950, 7943, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=90082 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[0, 4, 4, 29147, 0, 3617, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[0, 4, 4, 29147, 0, 3617, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 8274, 7944, 8273, 8273] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=96375 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[0, 4, 4, 29147, 0, 3617, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 8274, 7944, 8273, 8273] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=96375 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[8275, 8274, 8274, 8274, 8272, 7950, 7943, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=90082 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[32500, 25, 32764, 4, 0, 239, 0, 4] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[8275, 8274, 8274, 8274, 8272, 7950, 7943, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=90082 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[0, 4, 4, 29147, 0, 3617, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[32500, 25, 32764, 4, 0, 239, 0, 4] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[8275, 8274, 8274, 8274, 8272, 7950, 7943, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=90082 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[8272, 8275, 8273, 7947, 8273, 7946, 8275, 8275] avg=8192.0000 max=8275 max_rank=1 min=7946 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97377 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[0, 4, 4, 29147, 0, 3617, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[8275, 8274, 8274, 8274, 8272, 7950, 7943, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=90082 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[0, 4, 4, 29147, 0, 3617, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[32500, 25, 32764, 4, 0, 239, 0, 4] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[8272, 8275, 8273, 7947, 8273, 7946, 8275, 8275] avg=8192.0000 max=8275 max_rank=1 min=7946 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97377 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[32500, 25, 32764, 4, 0, 239, 0, 4] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[32500, 25, 32764, 4, 0, 239, 0, 4] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[8275, 8274, 8274, 8274, 8272, 7950, 7943, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=90082 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[8275, 8274, 8274, 8274, 8272, 7950, 7943, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=90082 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[8272, 8275, 8273, 7947, 8273, 7946, 8275, 8275] avg=8192.0000 max=8275 max_rank=1 min=7946 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97377 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[32764, 0, 4, 16, 1, 0, 1, 32750] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[8272, 8275, 8273, 7947, 8273, 7946, 8275, 8275] avg=8192.0000 max=8275 max_rank=1 min=7946 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97377 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[32500, 25, 32764, 4, 0, 239, 0, 4] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[32764, 0, 4, 16, 1, 0, 1, 32750] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[8272, 8275, 8273, 7947, 8273, 7946, 8275, 8275] avg=8192.0000 max=8275 max_rank=1 min=7946 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97377 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7949, 7941, 8274, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97868 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[8272, 8275, 8273, 7947, 8273, 7946, 8275, 8275] avg=8192.0000 max=8275 max_rank=1 min=7946 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97377 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[32500, 25, 32764, 4, 0, 239, 0, 4] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[32764, 0, 4, 16, 1, 0, 1, 32750] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7949, 7941, 8274, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97868 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[32500, 25, 32764, 4, 0, 239, 0, 4] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[32764, 0, 4, 16, 1, 0, 1, 32750] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[32764, 0, 4, 16, 1, 0, 1, 32750] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[8272, 8275, 8273, 7947, 8273, 7946, 8275, 8275] avg=8192.0000 max=8275 max_rank=1 min=7946 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97377 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7949, 7941, 8274, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97868 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[0, 1, 32757, 32764, 3, 4, 7, 0] avg=8192.0000 max=32764 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7949, 7941, 8274, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97868 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[8272, 8275, 8273, 7947, 8273, 7946, 8275, 8275] avg=8192.0000 max=8275 max_rank=1 min=7946 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97377 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[32764, 0, 4, 16, 1, 0, 1, 32750] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[0, 1, 32757, 32764, 3, 4, 7, 0] avg=8192.0000 max=32764 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7949, 7941, 8274, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97868 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[8275, 7944, 8273, 8272, 8275, 8274, 7949, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7949, 7941, 8274, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97868 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[8275, 7944, 8273, 8272, 8275, 8274, 7949, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[32764, 0, 4, 16, 1, 0, 1, 32750] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[0, 1, 32757, 32764, 3, 4, 7, 0] avg=8192.0000 max=32764 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[0, 1, 32757, 32764, 3, 4, 7, 0] avg=8192.0000 max=32764 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[32764, 0, 4, 16, 1, 0, 1, 32750] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[8275, 7944, 8273, 8272, 8275, 8274, 7949, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[4, 4, 0, 6, 32759, 2, 2, 32759] avg=8192.0000 max=32759 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[0, 1, 32757, 32764, 3, 4, 7, 0] avg=8192.0000 max=32764 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7949, 7941, 8274, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97868 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[8275, 7944, 8273, 8272, 8275, 8274, 7949, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[4, 4, 0, 6, 32759, 2, 2, 32759] avg=8192.0000 max=32759 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7949, 7941, 8274, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97868 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[0, 1, 32757, 32764, 3, 4, 7, 0] avg=8192.0000 max=32764 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[8275, 8274, 8272, 7950, 8275, 8275, 7941, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24484 max_load_reduction_pct=0.747398 moved_to_replicas=97923 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[8275, 7944, 8273, 8272, 8275, 8274, 7949, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[8275, 8274, 8272, 7950, 8275, 8275, 7941, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24484 max_load_reduction_pct=0.747398 moved_to_replicas=97923 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[4, 4, 0, 6, 32759, 2, 2, 32759] avg=8192.0000 max=32759 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[8275, 7944, 8273, 8272, 8275, 8274, 7949, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[4, 4, 0, 6, 32759, 2, 2, 32759] avg=8192.0000 max=32759 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[0, 1, 32757, 32764, 3, 4, 7, 0] avg=8192.0000 max=32764 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[0, 1, 32757, 32764, 3, 4, 7, 0] avg=8192.0000 max=32764 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[8275, 8274, 8272, 7950, 8275, 8275, 7941, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24484 max_load_reduction_pct=0.747398 moved_to_replicas=97923 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[4, 4, 0, 6, 32759, 2, 2, 32759] avg=8192.0000 max=32759 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[8275, 8274, 8272, 7950, 8275, 8275, 7941, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24484 max_load_reduction_pct=0.747398 moved_to_replicas=97923 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[4, 4, 0, 6, 32759, 2, 2, 32759] avg=8192.0000 max=32759 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[8275, 7944, 8273, 8272, 8275, 8274, 7949, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[8275, 7944, 8273, 8272, 8275, 8274, 7949, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[8275, 8274, 8272, 7950, 8275, 8275, 7941, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24484 max_load_reduction_pct=0.747398 moved_to_replicas=97923 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[8275, 8274, 8272, 7950, 8275, 8275, 7941, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24484 max_load_reduction_pct=0.747398 moved_to_replicas=97923 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[4, 4, 0, 6, 32759, 2, 2, 32759] avg=8192.0000 max=32759 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[4, 4, 0, 6, 32759, 2, 2, 32759] avg=8192.0000 max=32759 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[8275, 8274, 8272, 7950, 8275, 8275, 7941, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24484 max_load_reduction_pct=0.747398 moved_to_replicas=97923 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[8275, 8274, 8272, 7950, 8275, 8275, 7941, 8274] avg=8192.0000 max=8275 max_rank=0 min=7941 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24484 max_load_reduction_pct=0.747398 moved_to_replicas=97923 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=32 total_expected_recv=257025 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=67328 peak_demand=9404 required_capacity=1.147949 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=32 total_expected_recv=270273 total_dropped_tokens=455 drop_rate_pct=0.168348 records_with_drop_pct=3.125000 peak_rank_overflow=455 total_slack_slots=54144 peak_demand=10695 required_capacity=1.305542 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=32 total_expected_recv=267582 total_dropped_tokens=2217 drop_rate_pct=0.828531 records_with_drop_pct=6.250000 peak_rank_overflow=1422 total_slack_slots=57984 peak_demand=11662 required_capacity=1.423584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=32 total_expected_recv=243646 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=79488 peak_demand=8801 required_capacity=1.074341 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=32 total_expected_recv=264952 total_dropped_tokens=1878 drop_rate_pct=0.708808 records_with_drop_pct=6.250000 peak_rank_overflow=1423 total_slack_slots=60928 peak_demand=11663 required_capacity=1.423706 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=32 total_expected_recv=257076 total_dropped_tokens=792 drop_rate_pct=0.308080 records_with_drop_pct=3.125000 peak_rank_overflow=792 total_slack_slots=68096 peak_demand=11032 required_capacity=1.346680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=32 total_expected_recv=267279 total_dropped_tokens=2217 drop_rate_pct=0.829470 records_with_drop_pct=6.250000 peak_rank_overflow=1422 total_slack_slots=58752 peak_demand=11662 required_capacity=1.423584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=32 total_expected_recv=269319 total_dropped_tokens=2217 drop_rate_pct=0.823187 records_with_drop_pct=6.250000 peak_rank_overflow=1423 total_slack_slots=57216 peak_demand=11663 required_capacity=1.423706 + [2026-08-30 05:08:06] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 886.9 | throughput per GPU (TFLOP/s/GPU): 375.9 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.251316E+00 | loss scale: 1.0 | grad norm: 4.743 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 10240 | min safe capacity: 1.2500 | +[after training is done] datetime: 2026-08-30 05:08:06 +[rank0]:[W830 05:08:07.127668111 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W830 05:08:08.197999441 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W830 05:08:09.285924084 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W830 05:08:09.299451120 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W830 05:08:09.299883959 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W830 05:08:09.299954829 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 05:08:09.443000 478477 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 05:08:09.444000 478477 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 478569 closing signal SIGTERM +W0830 05:08:09.444000 478477 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 478570 closing signal SIGTERM +W0830 05:08:09.445000 478477 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 478571 closing signal SIGTERM +W0830 05:08:09.446000 478477 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 478572 closing signal SIGTERM +W0830 05:08:09.447000 478477 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 478573 closing signal SIGTERM +W0830 05:08:09.447000 478477 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 478574 closing signal SIGTERM +W0830 05:08:09.448000 478477 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 478575 closing signal SIGTERM +W0830 05:08:09.449000 478477 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 478576 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 478477 got signal: 15 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..4a59673190ade386d822d3a7bfd36210884ecc5e --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 819 +1, 819 +2, 819 +3, 819 +4, 819 +5, 819 +6, 819 +7, 819 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 2231 +1, 2231 +2, 2231 +3, 2231 +4, 2231 +5, 2231 +6, 2231 +7, 2231 +0, 2903 +1, 2903 +2, 2901 +3, 2895 +4, 2903 +5, 2903 +6, 2903 +7, 2903 +0, 42699 +1, 42701 +2, 42651 +3, 42691 +4, 42683 +5, 42715 +6, 42691 +7, 42725 +0, 43433 +1, 43445 +2, 43421 +3, 43435 +4, 43437 +5, 43417 +6, 43473 +7, 43445 +0, 44133 +1, 44131 +2, 44131 +3, 44131 +4, 44131 +5, 44131 +6, 44131 +7, 44131 +0, 44133 +1, 44131 +2, 44131 +3, 44131 +4, 44131 +5, 44131 +6, 44131 +7, 44131 +0, 45351 +1, 45349 +2, 45349 +3, 45349 +4, 45349 +5, 45349 +6, 45349 +7, 45349 +0, 45351 +1, 45349 +2, 45349 +3, 45349 +4, 45349 +5, 45349 +6, 45349 +7, 45349 +0, 75333 +1, 75331 +2, 75331 +3, 75331 +4, 75331 +5, 75331 +6, 75331 +7, 75331 +0, 75739 +1, 75331 +2, 75331 +3, 75331 +4, 75331 +5, 75331 +6, 75331 +7, 75331 +0, 76551 +1, 76143 +2, 76143 +3, 76143 +4, 76143 +5, 76143 +6, 76143 +7, 76143 +0, 87535 +1, 87585 +2, 87597 +3, 87593 +4, 87565 +5, 87619 +6, 87639 +7, 87615 +0, 89213 +1, 89303 +2, 89303 +3, 89303 +4, 89223 +5, 89303 +6, 89303 +7, 89303 +0, 93655 +1, 93739 +2, 93761 +3, 93757 +4, 93723 +5, 93769 +6, 93815 +7, 93807 +0, 113545 +1, 113767 +2, 113767 +3, 113767 +4, 113667 +5, 113767 +6, 113767 +7, 113767 +0, 113545 +1, 113767 +2, 113767 +3, 113767 +4, 113667 +5, 113767 +6, 113767 +7, 113767 +0, 113545 +1, 113767 +2, 113767 +3, 113767 +4, 113667 +5, 113767 +6, 113767 +7, 113767 +0, 113545 +1, 113767 +2, 113767 +3, 113767 +4, 113667 +5, 113767 +6, 113767 +7, 113767 +0, 113545 +1, 113767 +2, 113767 +3, 113767 +4, 113667 +5, 113767 +6, 113767 +7, 113767 +0, 113545 +1, 113767 +2, 113767 +3, 113767 +4, 113667 +5, 113767 +6, 113767 +7, 113767 +0, 109775 +1, 109675 +2, 109661 +3, 109647 +4, 109545 +5, 109653 +6, 109755 +7, 110251 +0, 1 +1, 4 +2, 4 +3, 196 +4, 109545 +5, 964 +6, 4 +7, 4 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1/run.env b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..d32e0201ae1703bee68537ea1808e89c2067df79 --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1/run.env @@ -0,0 +1,70 @@ +backend=ultraep +config=G1 +family=tierA +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=14336 +num_experts=8 +topk=2 +experts_per_rank=1 +expert_params=176160768 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=8192 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=1 +capacity_mode=static +capacity=1.8125 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=1024 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1/train.log b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..e338fe8ad81003a96120daf91d59d2126fc60390 --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1/train.log @@ -0,0 +1,8829 @@ +[05:08:13] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + G1 / ultraep (slots/rank=1 policy=matched) + E=8 K=2 H'=14336 m=1 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=static cap=1.8125 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0830 05:08:15.319000 481867 torch/distributed/run.py:766] +W0830 05:08:15.319000 481867 torch/distributed/run.py:766] ***************************************** +W0830 05:08:15.319000 481867 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 05:08:15.319000 481867 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 14336 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 1 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.8125 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 2 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 8 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank [Gloo] Rank [Gloo] Rank 15[Gloo] Rank [Gloo] Rank 2 is connected to is connected to 3 is connected to 707 is connected to 7 peer ranks. is connected to peer ranks. 7 peer ranks. Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : 7 peer ranks. 7Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : + +77 + + +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7[Gloo] Rank is connected to 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W830 05:08:33.234627902 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 05:08:33.234720261 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 05:08:33.234760046 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 05:08:33.234791515 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 05:08:33.234874670 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 05:08:33.234911097 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 05:08:33.234982442 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 05:08:33.394996763 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.924 seconds +time to initialize megatron (seconds): 7.479 +[after megatron is initialized] datetime: 2026-08-30 05:08:39 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 6720983040 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (41988096 elements, 41988096 padded size): + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.final_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_proj.weight +Params for bucket 3 (41984000 elements, 41984000 padded size): + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight +Params for bucket 4 (41984000 elements, 41984000 padded size): + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_proj.weight +Params for bucket 5 (41984000 elements, 41984000 padded size): + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight +Params for bucket 6 (41984000 elements, 41984000 padded size): + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_qkv.weight +Params for bucket 7 (41984000 elements, 41984000 padded size): + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight +Params for bucket 8 (41984000 elements, 41984000 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.pre_mlp_layernorm.weight +Params for bucket 9 (41984000 elements, 41984000 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.pre_mlp_layernorm.weight +Params for bucket 10 (41984000 elements, 41984000 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.pre_mlp_layernorm.weight +Params for bucket 11 (41984000 elements, 41984000 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.pre_mlp_layernorm.weight +Params for bucket 12 (41984000 elements, 41984000 padded size): + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_qkv.weight +Params for bucket 13 (41984000 elements, 41984000 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.pre_mlp_layernorm.weight +Params for bucket 14 (41984000 elements, 41984000 padded size): + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight +Params for bucket 15 (41984000 elements, 41984000 padded size): + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.self_attention.linear_qkv.weight +Params for bucket 16 (41984000 elements, 41984000 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.pre_mlp_layernorm.weight +Params for bucket 17 (41984000 elements, 41984000 padded size): + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_proj.weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 32 +Params for bucket 1 (58720256 elements, 58720256 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 +Params for bucket 2 (117440512 elements, 117440512 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 +Params for bucket 3 (58720256 elements, 58720256 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 +Params for bucket 4 (117440512 elements, 117440512 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 +Params for bucket 5 (58720256 elements, 58720256 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 +Params for bucket 6 (117440512 elements, 117440512 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 +Params for bucket 7 (58720256 elements, 58720256 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 +Params for bucket 8 (117440512 elements, 117440512 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 +Params for bucket 9 (58720256 elements, 58720256 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 +Params for bucket 10 (117440512 elements, 117440512 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 +Params for bucket 11 (58720256 elements, 58720256 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 +Params for bucket 12 (117440512 elements, 117440512 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 13 (58720256 elements, 58720256 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 +Params for bucket 14 (117440512 elements, 117440512 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 15 (58720256 elements, 58720256 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 +Params for bucket 16 (117440512 elements, 117440512 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 17 (58720256 elements, 58720256 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 18 (117440512 elements, 117440512 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 +Params for bucket 19 (58720256 elements, 58720256 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 +Params for bucket 20 (117440512 elements, 117440512 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 +Params for bucket 21 (58720256 elements, 58720256 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 +Params for bucket 22 (117440512 elements, 117440512 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 +Params for bucket 23 (58720256 elements, 58720256 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 +Params for bucket 24 (117440512 elements, 117440512 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 25 (58720256 elements, 58720256 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 +Params for bucket 26 (117440512 elements, 117440512 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 27 (58720256 elements, 58720256 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 +Params for bucket 28 (117440512 elements, 117440512 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 29 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 +Params for bucket 30 (117440512 elements, 117440512 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +Params for bucket 31 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 +Params for bucket 32 (117440512 elements, 117440512 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 05:08:42 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 192 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 05:08:42 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3218.41, 3221.08) + train/valid/test-data-iterators-setup ..........: (73.83, 93.44) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-30 05:08:42 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=14848 expected_recv=16797 expected_overflow=0 exact_recv_per_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=14848 expected_recv=16797 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=14848 expected_recv=16279 expected_overflow=0 exact_recv_per_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=14848 expected_recv=14029 expected_overflow=0 exact_recv_per_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=14848 expected_recv=17922 expected_overflow=0 exact_recv_per_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=14848 expected_recv=16798 expected_overflow=0 exact_recv_per_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=14848 expected_recv=16779 expected_overflow=0 exact_recv_per_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=14848 expected_recv=17922 actual_valid_recv=18048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=14848 expected_recv=16798 expected_overflow=0 exact_recv_per_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=14848 expected_recv=16279 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=14848 expected_recv=14029 actual_valid_recv=14208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=18048 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=14208 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=14848 expected_recv=16796 expected_overflow=0 exact_recv_per_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=14848 expected_recv=16798 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=14848 expected_recv=16779 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=14848 expected_recv=16798 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=14848 expected_recv=17043 expected_overflow=0 exact_recv_per_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=14848 expected_recv=16796 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=14848 expected_recv=17043 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=14848 expected_recv=18472 expected_overflow=0 exact_recv_per_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=14848 expected_recv=13506 expected_overflow=0 exact_recv_per_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=17152 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=14848 expected_recv=18472 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=14848 expected_recv=13506 actual_valid_recv=13696 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=18560 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=14848 expected_recv=14765 expected_overflow=0 exact_recv_per_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=13696 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=14848 expected_recv=14765 actual_valid_recv=14976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=14848 expected_recv=18473 expected_overflow=0 exact_recv_per_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=14848 expected_recv=16735 expected_overflow=0 exact_recv_per_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=14848 expected_recv=16424 expected_overflow=0 exact_recv_per_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=14976 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=14848 expected_recv=18473 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=14848 expected_recv=16735 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=14848 expected_recv=16822 expected_overflow=0 exact_recv_per_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=14848 expected_recv=16424 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=14848 expected_recv=18472 expected_overflow=0 exact_recv_per_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=14848 expected_recv=17563 expected_overflow=0 exact_recv_per_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=18560 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=14848 expected_recv=16822 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=14848 expected_recv=17563 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=14848 expected_recv=18472 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=17664 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=18560 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=14848 expected_recv=16021 expected_overflow=0 exact_recv_per_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=14848 expected_recv=16021 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=14848 expected_recv=17349 expected_overflow=0 exact_recv_per_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=14848 expected_recv=17561 expected_overflow=0 exact_recv_per_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=14848 expected_recv=17922 expected_overflow=0 exact_recv_per_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=14848 expected_recv=16512 expected_overflow=0 exact_recv_per_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=14848 expected_recv=17349 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=14848 expected_recv=17922 actual_valid_recv=18048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=14848 expected_recv=17561 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=14848 expected_recv=16854 expected_overflow=0 exact_recv_per_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=17536 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=14848 expected_recv=16512 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=18048 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=17664 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=14848 expected_recv=16854 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=17152 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=14848 expected_recv=15990 expected_overflow=0 exact_recv_per_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=14848 expected_recv=15479 expected_overflow=0 exact_recv_per_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=14848 expected_recv=16051 expected_overflow=0 exact_recv_per_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=14848 expected_recv=13971 expected_overflow=0 exact_recv_per_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=14848 expected_recv=16450 expected_overflow=0 exact_recv_per_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=14848 expected_recv=17095 expected_overflow=0 exact_recv_per_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=14848 expected_recv=15990 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=14848 expected_recv=15479 actual_valid_recv=15616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=14848 expected_recv=16051 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=14848 expected_recv=13971 actual_valid_recv=14208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=14848 expected_recv=16450 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=14848 expected_recv=17095 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=15616 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=14208 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=17408 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=14848 expected_recv=15871 expected_overflow=0 exact_recv_per_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=14848 expected_recv=15871 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=14848 expected_recv=19640 expected_overflow=0 exact_recv_per_rank=[10791, 7082, 3420, 3458, 9018, 10792, 10183, 10792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=14848 expected_recv=15990 expected_overflow=0 exact_recv_per_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=14848 expected_recv=16163 expected_overflow=0 exact_recv_per_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=14848 expected_recv=21505 expected_overflow=0 exact_recv_per_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=14848 expected_recv=11030 expected_overflow=0 exact_recv_per_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=14848 expected_recv=16140 expected_overflow=0 exact_recv_per_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=14848 expected_recv=19640 actual_valid_recv=19840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=14848 expected_recv=15990 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=14848 expected_recv=16163 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=14848 expected_recv=21505 actual_valid_recv=21632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=14848 expected_recv=11030 actual_valid_recv=11264 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=19840 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=14848 expected_recv=16140 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=11264 slack_slots=18432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=21632 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=14848 expected_recv=17301 expected_overflow=0 exact_recv_per_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=14848 expected_recv=17301 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=14848 expected_recv=20307 expected_overflow=0 exact_recv_per_rank=[10869, 5599, 10870, 6743, 8738, 5098, 6749, 10870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=17664 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=14848 expected_recv=15666 expected_overflow=0 exact_recv_per_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=14848 expected_recv=11721 expected_overflow=0 exact_recv_per_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=14848 expected_recv=19643 expected_overflow=0 exact_recv_per_rank=[10791, 7082, 3420, 3458, 9018, 10792, 10183, 10792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=14848 expected_recv=21503 expected_overflow=0 exact_recv_per_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=14848 expected_recv=20307 actual_valid_recv=20352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=14848 expected_recv=19642 expected_overflow=0 exact_recv_per_rank=[10791, 7082, 3420, 3458, 9018, 10792, 10183, 10792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=14848 expected_recv=15666 actual_valid_recv=15744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=14848 expected_recv=11721 actual_valid_recv=11776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=14848 expected_recv=19643 actual_valid_recv=19840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=14848 expected_recv=21503 actual_valid_recv=21632 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=20352 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=14848 expected_recv=19642 actual_valid_recv=19840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=15744 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=11776 slack_slots=17920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=19840 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=21632 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=19840 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=14848 expected_recv=12268 expected_overflow=0 exact_recv_per_rank=[10791, 7082, 3420, 3458, 9018, 10792, 10183, 10792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=14848 expected_recv=12268 actual_valid_recv=12416 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=14848 expected_recv=16796 expected_overflow=0 exact_recv_per_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=12416 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=14848 expected_recv=17681 expected_overflow=0 exact_recv_per_rank=[7452, 9179, 7886, 9181, 6651, 8533, 7472, 9182] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=14848 expected_recv=19033 expected_overflow=0 exact_recv_per_rank=[10791, 7082, 3420, 3458, 9018, 10792, 10183, 10792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=14848 expected_recv=14056 expected_overflow=0 exact_recv_per_rank=[10869, 5599, 10870, 6743, 8738, 5098, 6749, 10870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=14848 expected_recv=10300 expected_overflow=0 exact_recv_per_rank=[10791, 7082, 3420, 3458, 9018, 10792, 10183, 10792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=14848 expected_recv=16337 expected_overflow=0 exact_recv_per_rank=[10791, 7082, 3420, 3458, 9018, 10792, 10183, 10792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=14848 expected_recv=17681 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=14848 expected_recv=20309 expected_overflow=0 exact_recv_per_rank=[10869, 5599, 10870, 6743, 8738, 5098, 6749, 10870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=14848 expected_recv=19033 actual_valid_recv=19200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=14848 expected_recv=14056 actual_valid_recv=14336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=14848 expected_recv=10300 actual_valid_recv=10624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=17792 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=14848 expected_recv=16337 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=14848 expected_recv=20309 actual_valid_recv=20352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=19200 slack_slots=10496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=14336 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=14848 expected_recv=16796 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=10624 slack_slots=19072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=20352 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=14848 expected_recv=20308 expected_overflow=0 exact_recv_per_rank=[10869, 5599, 10870, 6743, 8738, 5098, 6749, 10870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=14848 expected_recv=20308 actual_valid_recv=20352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=20352 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=14848 expected_recv=19054 expected_overflow=0 exact_recv_per_rank=[8697, 8696, 8696, 5930, 7469, 8695, 8696, 8657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=14848 expected_recv=14713 expected_overflow=0 exact_recv_per_rank=[10869, 5599, 10870, 6743, 8738, 5098, 6749, 10870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=14848 expected_recv=13778 expected_overflow=0 exact_recv_per_rank=[7452, 9179, 7886, 9181, 6651, 8533, 7472, 9182] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=14848 expected_recv=19054 actual_valid_recv=19072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=14848 expected_recv=14713 actual_valid_recv=14976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=14848 expected_recv=14231 expected_overflow=0 exact_recv_per_rank=[10869, 5599, 10870, 6743, 8738, 5098, 6749, 10870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=14848 expected_recv=15017 expected_overflow=0 exact_recv_per_rank=[10869, 5599, 10870, 6743, 8738, 5098, 6749, 10870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=14848 expected_recv=15462 expected_overflow=0 exact_recv_per_rank=[7452, 9179, 7886, 9181, 6651, 8533, 7472, 9182] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=19072 slack_slots=10624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=14848 expected_recv=13778 actual_valid_recv=13952 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=14976 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=14848 expected_recv=14231 actual_valid_recv=14336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=14848 expected_recv=15017 actual_valid_recv=15104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=14848 expected_recv=15462 actual_valid_recv=15616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=13952 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=14336 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=14848 expected_recv=11271 expected_overflow=0 exact_recv_per_rank=[7452, 9179, 7886, 9181, 6651, 8533, 7472, 9182] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=15104 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=15616 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=14848 expected_recv=12727 expected_overflow=0 exact_recv_per_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=14848 expected_recv=11271 actual_valid_recv=11392 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=11392 slack_slots=18304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=14848 expected_recv=12727 actual_valid_recv=12928 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=14848 expected_recv=18847 expected_overflow=0 exact_recv_per_rank=[8724, 7978, 7424, 7164, 8724, 8725, 8073, 8724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=14848 expected_recv=17061 expected_overflow=0 exact_recv_per_rank=[7452, 9179, 7886, 9181, 6651, 8533, 7472, 9182] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=14848 expected_recv=18364 expected_overflow=0 exact_recv_per_rank=[8697, 8696, 8696, 5930, 7469, 8695, 8696, 8657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=12928 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=14848 expected_recv=18847 actual_valid_recv=19072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=14848 expected_recv=15023 expected_overflow=0 exact_recv_per_rank=[7452, 9179, 7886, 9181, 6651, 8533, 7472, 9182] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=14848 expected_recv=17061 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=14848 expected_recv=20399 expected_overflow=0 exact_recv_per_rank=[7452, 9179, 7886, 9181, 6651, 8533, 7472, 9182] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=14848 expected_recv=15783 expected_overflow=0 exact_recv_per_rank=[8697, 8696, 8696, 5930, 7469, 8695, 8696, 8657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=14848 expected_recv=18364 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=19072 slack_slots=10624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=14848 expected_recv=15023 actual_valid_recv=15232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=17152 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=14848 expected_recv=20399 actual_valid_recv=20480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=14848 expected_recv=15783 actual_valid_recv=15872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=18560 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=14848 expected_recv=16785 expected_overflow=0 exact_recv_per_rank=[8697, 8696, 8696, 5930, 7469, 8695, 8696, 8657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=15232 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=20480 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=15872 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=14848 expected_recv=16785 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=14848 expected_recv=11859 expected_overflow=0 exact_recv_per_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=14848 expected_recv=11859 actual_valid_recv=12288 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=14848 expected_recv=19881 expected_overflow=0 exact_recv_per_rank=[8697, 8696, 8696, 5930, 7469, 8695, 8696, 8657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=14848 expected_recv=17499 expected_overflow=0 exact_recv_per_rank=[8686, 8461, 7702, 8685, 6917, 8686, 7712, 8687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=14848 expected_recv=18847 expected_overflow=0 exact_recv_per_rank=[8724, 7978, 7424, 7164, 8724, 8725, 8073, 8724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=12288 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=14848 expected_recv=10563 expected_overflow=0 exact_recv_per_rank=[8697, 8696, 8696, 5930, 7469, 8695, 8696, 8657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=14848 expected_recv=19881 actual_valid_recv=19968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=14848 expected_recv=17499 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=14848 expected_recv=10762 expected_overflow=0 exact_recv_per_rank=[8697, 8696, 8696, 5930, 7469, 8695, 8696, 8657] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=14848 expected_recv=18847 expected_overflow=0 exact_recv_per_rank=[8724, 7978, 7424, 7164, 8724, 8725, 8073, 8724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=14848 expected_recv=18847 actual_valid_recv=19072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=19968 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=14848 expected_recv=10563 actual_valid_recv=10752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=14848 expected_recv=10330 expected_overflow=0 exact_recv_per_rank=[8724, 7978, 7424, 7164, 8724, 8725, 8073, 8724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=17536 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=14848 expected_recv=10762 actual_valid_recv=10880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=19072 slack_slots=10624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=14848 expected_recv=18847 actual_valid_recv=19072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=10752 slack_slots=18944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=14848 expected_recv=10330 actual_valid_recv=10496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=10880 slack_slots=18816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=19072 slack_slots=10624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=10496 slack_slots=19200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=14848 expected_recv=11987 expected_overflow=0 exact_recv_per_rank=[8724, 7978, 7424, 7164, 8724, 8725, 8073, 8724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=14848 expected_recv=23192 expected_overflow=0 exact_recv_per_rank=[14061, 5050, 1629, 14061, 5647, 4245, 7567, 13276] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=14848 expected_recv=17500 expected_overflow=0 exact_recv_per_rank=[8686, 8461, 7702, 8685, 6917, 8686, 7712, 8687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=14848 expected_recv=17835 expected_overflow=0 exact_recv_per_rank=[8724, 7978, 7424, 7164, 8724, 8725, 8073, 8724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=14848 expected_recv=17236 expected_overflow=0 exact_recv_per_rank=[8724, 7978, 7424, 7164, 8724, 8725, 8073, 8724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=14848 expected_recv=11987 actual_valid_recv=12160 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=14848 expected_recv=17501 expected_overflow=0 exact_recv_per_rank=[8686, 8461, 7702, 8685, 6917, 8686, 7712, 8687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=14848 expected_recv=23192 actual_valid_recv=23296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=14848 expected_recv=17500 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=14848 expected_recv=11587 expected_overflow=0 exact_recv_per_rank=[8686, 8461, 7702, 8685, 6917, 8686, 7712, 8687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=14848 expected_recv=17835 actual_valid_recv=18048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=23296 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=12160 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=14848 expected_recv=17236 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=14848 expected_recv=17501 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=17664 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=14848 expected_recv=11587 actual_valid_recv=11776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=18048 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=17792 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=17280 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=11776 slack_slots=17920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=14848 expected_recv=17920 expected_overflow=0 exact_recv_per_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=14848 expected_recv=14428 expected_overflow=0 exact_recv_per_rank=[5958, 8244, 8634, 8634, 8635, 8633, 8634, 8164] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=14848 expected_recv=16525 expected_overflow=0 exact_recv_per_rank=[8686, 8461, 7702, 8685, 6917, 8686, 7712, 8687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=14848 expected_recv=17920 actual_valid_recv=18048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=14848 expected_recv=11608 expected_overflow=0 exact_recv_per_rank=[14061, 5050, 1629, 14061, 5647, 4245, 7567, 13276] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=14848 expected_recv=14428 actual_valid_recv=14720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=14848 expected_recv=16525 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=14848 expected_recv=15729 expected_overflow=0 exact_recv_per_rank=[8686, 8461, 7702, 8685, 6917, 8686, 7712, 8687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=14848 expected_recv=21034 expected_overflow=0 exact_recv_per_rank=[14061, 5050, 1629, 14061, 5647, 4245, 7567, 13276] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=18048 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=14848 expected_recv=11608 actual_valid_recv=11904 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=14848 expected_recv=17500 expected_overflow=0 exact_recv_per_rank=[8686, 8461, 7702, 8685, 6917, 8686, 7712, 8687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=14720 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=14848 expected_recv=10760 expected_overflow=0 exact_recv_per_rank=[14061, 5050, 1629, 14061, 5647, 4245, 7567, 13276] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=14848 expected_recv=15729 actual_valid_recv=16000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=14848 expected_recv=21034 actual_valid_recv=21248 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=11904 slack_slots=17792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=14848 expected_recv=17500 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=14848 expected_recv=10760 actual_valid_recv=10880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16000 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=21248 slack_slots=8448 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=17536 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=10880 slack_slots=18816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=14848 expected_recv=16867 expected_overflow=0 exact_recv_per_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=14848 expected_recv=15655 expected_overflow=0 exact_recv_per_rank=[7156, 6995, 10896, 9224, 6332, 10896, 10896, 3141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=14848 expected_recv=16867 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=14848 expected_recv=12796 expected_overflow=0 exact_recv_per_rank=[14061, 5050, 1629, 14061, 5647, 4245, 7567, 13276] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=14848 expected_recv=15655 actual_valid_recv=15872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=14848 expected_recv=17103 expected_overflow=0 exact_recv_per_rank=[5958, 8244, 8634, 8634, 8635, 8633, 8634, 8164] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=14848 expected_recv=14308 expected_overflow=0 exact_recv_per_rank=[14061, 5050, 1629, 14061, 5647, 4245, 7567, 13276] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=14848 expected_recv=16634 expected_overflow=0 exact_recv_per_rank=[5958, 8244, 8634, 8634, 8635, 8633, 8634, 8164] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=14848 expected_recv=12796 actual_valid_recv=12928 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=15872 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=14848 expected_recv=17103 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=14848 expected_recv=23192 expected_overflow=0 exact_recv_per_rank=[14061, 5050, 1629, 14061, 5647, 4245, 7567, 13276] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=14848 expected_recv=16886 expected_overflow=0 exact_recv_per_rank=[5958, 8244, 8634, 8634, 8635, 8633, 8634, 8164] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=14848 expected_recv=14308 actual_valid_recv=14464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=14848 expected_recv=16634 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=12928 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=17280 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=14848 expected_recv=23192 actual_valid_recv=23296 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=14464 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=14848 expected_recv=16886 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=23296 slack_slots=6400 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=14848 expected_recv=14209 expected_overflow=0 exact_recv_per_rank=[10791, 7082, 3420, 3458, 9018, 10792, 10183, 10792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=14848 expected_recv=16650 expected_overflow=0 exact_recv_per_rank=[8376, 8481, 8480, 8479, 7995, 6765, 8480, 8480] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=14848 expected_recv=14209 actual_valid_recv=14592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=14848 expected_recv=15098 expected_overflow=0 exact_recv_per_rank=[5958, 8244, 8634, 8634, 8635, 8633, 8634, 8164] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=14848 expected_recv=19396 expected_overflow=0 exact_recv_per_rank=[7156, 6995, 10896, 9224, 6332, 10896, 10896, 3141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=14848 expected_recv=16650 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=14592 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=14848 expected_recv=17105 expected_overflow=0 exact_recv_per_rank=[5958, 8244, 8634, 8634, 8635, 8633, 8634, 8164] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=14848 expected_recv=11566 expected_overflow=0 exact_recv_per_rank=[7156, 6995, 10896, 9224, 6332, 10896, 10896, 3141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=14848 expected_recv=15098 actual_valid_recv=15360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=14848 expected_recv=17104 expected_overflow=0 exact_recv_per_rank=[5958, 8244, 8634, 8634, 8635, 8633, 8634, 8164] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=14848 expected_recv=19396 actual_valid_recv=19584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=14848 expected_recv=18923 expected_overflow=0 exact_recv_per_rank=[7156, 6995, 10896, 9224, 6332, 10896, 10896, 3141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=14848 expected_recv=17105 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=15360 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=14848 expected_recv=11566 actual_valid_recv=11776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=14848 expected_recv=17104 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=19584 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=17280 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=14848 expected_recv=18923 actual_valid_recv=19072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=11776 slack_slots=17920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=17280 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=19072 slack_slots=10624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=14848 expected_recv=12131 expected_overflow=0 exact_recv_per_rank=[10869, 5599, 10870, 6743, 8738, 5098, 6749, 10870] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=14848 expected_recv=17485 expected_overflow=0 exact_recv_per_rank=[7156, 6995, 10896, 9224, 6332, 10896, 10896, 3141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=14848 expected_recv=12131 actual_valid_recv=12288 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=14848 expected_recv=15296 expected_overflow=0 exact_recv_per_rank=[8376, 8481, 8480, 8479, 7995, 6765, 8480, 8480] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=14848 expected_recv=14831 expected_overflow=0 exact_recv_per_rank=[7156, 6995, 10896, 9224, 6332, 10896, 10896, 3141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=14848 expected_recv=17011 expected_overflow=0 exact_recv_per_rank=[8376, 8481, 8480, 8479, 7995, 6765, 8480, 8480] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=12288 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=14848 expected_recv=17485 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=14848 expected_recv=17723 expected_overflow=0 exact_recv_per_rank=[7156, 6995, 10896, 9224, 6332, 10896, 10896, 3141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=14848 expected_recv=14831 actual_valid_recv=15104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[991, 24884, 5241, 2569, 13382, 3150, 7068, 8251] avg=8192.0000 max=24884 max_rank=1 min=991 min_rank=0 imbalance_ratio=3.037598 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=14848 expected_recv=15296 actual_valid_recv=15360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=14848 expected_recv=15816 expected_overflow=0 exact_recv_per_rank=[8376, 8481, 8480, 8479, 7995, 6765, 8480, 8480] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=14848 expected_recv=17011 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=17792 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=15104 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=14848 expected_recv=17723 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=15360 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=17152 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=14848 expected_recv=15816 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=17920 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] avg=8192.0000 max=8524 max_rank=2 min=6025 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=16360 max_load_reduction_pct=0.657451 moved_to_replicas=41587 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=14848 expected_recv=20397 expected_overflow=0 exact_recv_per_rank=[7452, 9179, 7886, 9181, 6651, 8533, 7472, 9182] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=14848 expected_recv=20397 actual_valid_recv=20608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=14848 expected_recv=16453 expected_overflow=0 exact_recv_per_rank=[8376, 8481, 8480, 8479, 7995, 6765, 8480, 8480] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=14848 expected_recv=17012 expected_overflow=0 exact_recv_per_rank=[8376, 8481, 8480, 8479, 7995, 6765, 8480, 8480] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=20608 slack_slots=9088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=14848 expected_recv=15821 expected_overflow=0 exact_recv_per_rank=[8376, 8481, 8480, 8479, 7995, 6765, 8480, 8480] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=14848 expected_recv=16453 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=14848 expected_recv=17012 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=14848 expected_recv=15821 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=17280 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[991, 24884, 5241, 2569, 13382, 3150, 7068, 8251] avg=8192.0000 max=24884 max_rank=1 min=991 min_rank=0 imbalance_ratio=3.037598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[991, 24884, 5241, 2569, 13382, 3150, 7068, 8251] avg=8192.0000 max=24884 max_rank=1 min=991 min_rank=0 imbalance_ratio=3.037598 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[8387, 1691, 4070, 7203, 16186, 4440, 11688, 11871] avg=8192.0000 max=16186 max_rank=4 min=1691 min_rank=1 imbalance_ratio=1.975830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[991, 24884, 5241, 2569, 13382, 3150, 7068, 8251] avg=8192.0000 max=24884 max_rank=1 min=991 min_rank=0 imbalance_ratio=3.037598 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=14848 expected_recv=19880 expected_overflow=0 exact_recv_per_rank=[8697, 8696, 8696, 5930, 7469, 8695, 8696, 8657] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] avg=8192.0000 max=8524 max_rank=2 min=6025 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=16360 max_load_reduction_pct=0.657451 moved_to_replicas=41587 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] avg=8192.0000 max=8524 max_rank=2 min=6025 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=16360 max_load_reduction_pct=0.657451 moved_to_replicas=41587 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=14848 expected_recv=19880 actual_valid_recv=19968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] avg=8192.0000 max=8939 max_rank=4 min=7003 min_rank=2 imbalance_ratio=1.091187 max_load_reduction=7247 max_load_reduction_pct=0.447733 moved_to_replicas=22777 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] avg=8192.0000 max=8524 max_rank=2 min=6025 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=16360 max_load_reduction_pct=0.657451 moved_to_replicas=41587 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=19968 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[991, 24884, 5241, 2569, 13382, 3150, 7068, 8251] avg=8192.0000 max=24884 max_rank=1 min=991 min_rank=0 imbalance_ratio=3.037598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[991, 24884, 5241, 2569, 13382, 3150, 7068, 8251] avg=8192.0000 max=24884 max_rank=1 min=991 min_rank=0 imbalance_ratio=3.037598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[991, 24884, 5241, 2569, 13382, 3150, 7068, 8251] avg=8192.0000 max=24884 max_rank=1 min=991 min_rank=0 imbalance_ratio=3.037598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[8387, 1691, 4070, 7203, 16186, 4440, 11688, 11871] avg=8192.0000 max=16186 max_rank=4 min=1691 min_rank=1 imbalance_ratio=1.975830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[8387, 1691, 4070, 7203, 16186, 4440, 11688, 11871] avg=8192.0000 max=16186 max_rank=4 min=1691 min_rank=1 imbalance_ratio=1.975830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[8615, 1037, 7593, 20099, 902, 15779, 9023, 2488] avg=8192.0000 max=20099 max_rank=3 min=902 min_rank=4 imbalance_ratio=2.453491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[8387, 1691, 4070, 7203, 16186, 4440, 11688, 11871] avg=8192.0000 max=16186 max_rank=4 min=1691 min_rank=1 imbalance_ratio=1.975830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] avg=8192.0000 max=8524 max_rank=2 min=6025 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=16360 max_load_reduction_pct=0.657451 moved_to_replicas=41587 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=14848 expected_recv=17143 expected_overflow=0 exact_recv_per_rank=[8724, 7978, 7424, 7164, 8724, 8725, 8073, 8724] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] avg=8192.0000 max=8524 max_rank=2 min=6025 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=16360 max_load_reduction_pct=0.657451 moved_to_replicas=41587 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] avg=8192.0000 max=8524 max_rank=2 min=6025 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=16360 max_load_reduction_pct=0.657451 moved_to_replicas=41587 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] avg=8192.0000 max=8939 max_rank=4 min=7003 min_rank=2 imbalance_ratio=1.091187 max_load_reduction=7247 max_load_reduction_pct=0.447733 moved_to_replicas=22777 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=14848 expected_recv=17143 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] avg=8192.0000 max=8939 max_rank=4 min=7003 min_rank=2 imbalance_ratio=1.091187 max_load_reduction=7247 max_load_reduction_pct=0.447733 moved_to_replicas=22777 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=17280 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] avg=8192.0000 max=9135 max_rank=4 min=3770 min_rank=1 imbalance_ratio=1.115112 max_load_reduction=10964 max_load_reduction_pct=0.545500 moved_to_replicas=33906 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] avg=8192.0000 max=8939 max_rank=4 min=7003 min_rank=2 imbalance_ratio=1.091187 max_load_reduction=7247 max_load_reduction_pct=0.447733 moved_to_replicas=22777 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[8387, 1691, 4070, 7203, 16186, 4440, 11688, 11871] avg=8192.0000 max=16186 max_rank=4 min=1691 min_rank=1 imbalance_ratio=1.975830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[8387, 1691, 4070, 7203, 16186, 4440, 11688, 11871] avg=8192.0000 max=16186 max_rank=4 min=1691 min_rank=1 imbalance_ratio=1.975830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[8387, 1691, 4070, 7203, 16186, 4440, 11688, 11871] avg=8192.0000 max=16186 max_rank=4 min=1691 min_rank=1 imbalance_ratio=1.975830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[8615, 1037, 7593, 20099, 902, 15779, 9023, 2488] avg=8192.0000 max=20099 max_rank=3 min=902 min_rank=4 imbalance_ratio=2.453491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[8615, 1037, 7593, 20099, 902, 15779, 9023, 2488] avg=8192.0000 max=20099 max_rank=3 min=902 min_rank=4 imbalance_ratio=2.453491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[5823, 20298, 4359, 6463, 10601, 2673, 9588, 5731] avg=8192.0000 max=20298 max_rank=1 min=2673 min_rank=5 imbalance_ratio=2.477783 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=14848 expected_recv=17231 expected_overflow=0 exact_recv_per_rank=[8686, 8461, 7702, 8685, 6917, 8686, 7712, 8687] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[8615, 1037, 7593, 20099, 902, 15779, 9023, 2488] avg=8192.0000 max=20099 max_rank=3 min=902 min_rank=4 imbalance_ratio=2.453491 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=14848 expected_recv=17231 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] avg=8192.0000 max=8939 max_rank=4 min=7003 min_rank=2 imbalance_ratio=1.091187 max_load_reduction=7247 max_load_reduction_pct=0.447733 moved_to_replicas=22777 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=17408 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] avg=8192.0000 max=8939 max_rank=4 min=7003 min_rank=2 imbalance_ratio=1.091187 max_load_reduction=7247 max_load_reduction_pct=0.447733 moved_to_replicas=22777 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] avg=8192.0000 max=8939 max_rank=4 min=7003 min_rank=2 imbalance_ratio=1.091187 max_load_reduction=7247 max_load_reduction_pct=0.447733 moved_to_replicas=22777 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] avg=8192.0000 max=9135 max_rank=4 min=3770 min_rank=1 imbalance_ratio=1.115112 max_load_reduction=10964 max_load_reduction_pct=0.545500 moved_to_replicas=33906 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] avg=8192.0000 max=8275 max_rank=5 min=7777 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=12023 max_load_reduction_pct=0.592324 moved_to_replicas=30131 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] avg=8192.0000 max=9135 max_rank=4 min=3770 min_rank=1 imbalance_ratio=1.115112 max_load_reduction=10964 max_load_reduction_pct=0.545500 moved_to_replicas=33906 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] avg=8192.0000 max=9135 max_rank=4 min=3770 min_rank=1 imbalance_ratio=1.115112 max_load_reduction=10964 max_load_reduction_pct=0.545500 moved_to_replicas=33906 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[8615, 1037, 7593, 20099, 902, 15779, 9023, 2488] avg=8192.0000 max=20099 max_rank=3 min=902 min_rank=4 imbalance_ratio=2.453491 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=14848 expected_recv=14182 expected_overflow=0 exact_recv_per_rank=[14061, 5050, 1629, 14061, 5647, 4245, 7567, 13276] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[8615, 1037, 7593, 20099, 902, 15779, 9023, 2488] avg=8192.0000 max=20099 max_rank=3 min=902 min_rank=4 imbalance_ratio=2.453491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[5823, 20298, 4359, 6463, 10601, 2673, 9588, 5731] avg=8192.0000 max=20298 max_rank=1 min=2673 min_rank=5 imbalance_ratio=2.477783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[8615, 1037, 7593, 20099, 902, 15779, 9023, 2488] avg=8192.0000 max=20099 max_rank=3 min=902 min_rank=4 imbalance_ratio=2.453491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[12437, 13179, 4062, 7594, 7742, 9524, 5102, 5896] avg=8192.0000 max=13179 max_rank=1 min=4062 min_rank=2 imbalance_ratio=1.608765 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=14848 expected_recv=14182 actual_valid_recv=14336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[5823, 20298, 4359, 6463, 10601, 2673, 9588, 5731] avg=8192.0000 max=20298 max_rank=1 min=2673 min_rank=5 imbalance_ratio=2.477783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[5823, 20298, 4359, 6463, 10601, 2673, 9588, 5731] avg=8192.0000 max=20298 max_rank=1 min=2673 min_rank=5 imbalance_ratio=2.477783 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=14336 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] avg=8192.0000 max=9135 max_rank=4 min=3770 min_rank=1 imbalance_ratio=1.115112 max_load_reduction=10964 max_load_reduction_pct=0.545500 moved_to_replicas=33906 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] avg=8192.0000 max=9135 max_rank=4 min=3770 min_rank=1 imbalance_ratio=1.115112 max_load_reduction=10964 max_load_reduction_pct=0.545500 moved_to_replicas=33906 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] avg=8192.0000 max=8275 max_rank=5 min=7777 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=12023 max_load_reduction_pct=0.592324 moved_to_replicas=30131 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] avg=8192.0000 max=9525 max_rank=0 min=5896 min_rank=7 imbalance_ratio=1.162720 max_load_reduction=3654 max_load_reduction_pct=0.277259 moved_to_replicas=20495 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] avg=8192.0000 max=9135 max_rank=4 min=3770 min_rank=1 imbalance_ratio=1.115112 max_load_reduction=10964 max_load_reduction_pct=0.545500 moved_to_replicas=33906 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] avg=8192.0000 max=8275 max_rank=5 min=7777 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=12023 max_load_reduction_pct=0.592324 moved_to_replicas=30131 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] avg=8192.0000 max=8275 max_rank=5 min=7777 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=12023 max_load_reduction_pct=0.592324 moved_to_replicas=30131 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=14848 expected_recv=16714 expected_overflow=0 exact_recv_per_rank=[5958, 8244, 8634, 8634, 8635, 8633, 8634, 8164] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[5823, 20298, 4359, 6463, 10601, 2673, 9588, 5731] avg=8192.0000 max=20298 max_rank=1 min=2673 min_rank=5 imbalance_ratio=2.477783 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=14848 expected_recv=16714 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[5823, 20298, 4359, 6463, 10601, 2673, 9588, 5731] avg=8192.0000 max=20298 max_rank=1 min=2673 min_rank=5 imbalance_ratio=2.477783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 1685, 3449, 19120, 2980, 9656, 12376] avg=8192.0000 max=19120 max_rank=4 min=1685 min_rank=2 imbalance_ratio=2.333984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[12437, 13179, 4062, 7594, 7742, 9524, 5102, 5896] avg=8192.0000 max=13179 max_rank=1 min=4062 min_rank=2 imbalance_ratio=1.608765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[5823, 20298, 4359, 6463, 10601, 2673, 9588, 5731] avg=8192.0000 max=20298 max_rank=1 min=2673 min_rank=5 imbalance_ratio=2.477783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[12437, 13179, 4062, 7594, 7742, 9524, 5102, 5896] avg=8192.0000 max=13179 max_rank=1 min=4062 min_rank=2 imbalance_ratio=1.608765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[12437, 13179, 4062, 7594, 7742, 9524, 5102, 5896] avg=8192.0000 max=13179 max_rank=1 min=4062 min_rank=2 imbalance_ratio=1.608765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] avg=8192.0000 max=8275 max_rank=5 min=7777 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=12023 max_load_reduction_pct=0.592324 moved_to_replicas=30131 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] avg=8192.0000 max=8275 max_rank=5 min=7777 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=12023 max_load_reduction_pct=0.592324 moved_to_replicas=30131 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] avg=8192.0000 max=10404 max_rank=7 min=3449 min_rank=3 imbalance_ratio=1.270020 max_load_reduction=8716 max_load_reduction_pct=0.455858 moved_to_replicas=13521 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] avg=8192.0000 max=9525 max_rank=0 min=5896 min_rank=7 imbalance_ratio=1.162720 max_load_reduction=3654 max_load_reduction_pct=0.277259 moved_to_replicas=20495 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] avg=8192.0000 max=9525 max_rank=0 min=5896 min_rank=7 imbalance_ratio=1.162720 max_load_reduction=3654 max_load_reduction_pct=0.277259 moved_to_replicas=20495 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] avg=8192.0000 max=8275 max_rank=5 min=7777 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=12023 max_load_reduction_pct=0.592324 moved_to_replicas=30131 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] avg=8192.0000 max=9525 max_rank=0 min=5896 min_rank=7 imbalance_ratio=1.162720 max_load_reduction=3654 max_load_reduction_pct=0.277259 moved_to_replicas=20495 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=14848 expected_recv=15493 expected_overflow=0 exact_recv_per_rank=[7156, 6995, 10896, 9224, 6332, 10896, 10896, 3141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=14848 expected_recv=15493 actual_valid_recv=15616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[12437, 13179, 4062, 7594, 7742, 9524, 5102, 5896] avg=8192.0000 max=13179 max_rank=1 min=4062 min_rank=2 imbalance_ratio=1.608765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[12437, 13179, 4062, 7594, 7742, 9524, 5102, 5896] avg=8192.0000 max=13179 max_rank=1 min=4062 min_rank=2 imbalance_ratio=1.608765 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=15616 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[16321, 1552, 3420, 3458, 1222, 10792, 10183, 18588] avg=8192.0000 max=18588 max_rank=7 min=1222 min_rank=4 imbalance_ratio=2.269043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 1685, 3449, 19120, 2980, 9656, 12376] avg=8192.0000 max=19120 max_rank=4 min=1685 min_rank=2 imbalance_ratio=2.333984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 1685, 3449, 19120, 2980, 9656, 12376] avg=8192.0000 max=19120 max_rank=4 min=1685 min_rank=2 imbalance_ratio=2.333984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[12437, 13179, 4062, 7594, 7742, 9524, 5102, 5896] avg=8192.0000 max=13179 max_rank=1 min=4062 min_rank=2 imbalance_ratio=1.608765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 1685, 3449, 19120, 2980, 9656, 12376] avg=8192.0000 max=19120 max_rank=4 min=1685 min_rank=2 imbalance_ratio=2.333984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] avg=8192.0000 max=9525 max_rank=0 min=5896 min_rank=7 imbalance_ratio=1.162720 max_load_reduction=3654 max_load_reduction_pct=0.277259 moved_to_replicas=20495 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] avg=8192.0000 max=9525 max_rank=0 min=5896 min_rank=7 imbalance_ratio=1.162720 max_load_reduction=3654 max_load_reduction_pct=0.277259 moved_to_replicas=20495 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[10791, 7082, 3420, 3458, 9018, 10792, 10183, 10792] avg=8192.0000 max=10792 max_rank=5 min=3420 min_rank=2 imbalance_ratio=1.317383 max_load_reduction=7796 max_load_reduction_pct=0.419410 moved_to_replicas=32770 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] avg=8192.0000 max=10404 max_rank=7 min=3449 min_rank=3 imbalance_ratio=1.270020 max_load_reduction=8716 max_load_reduction_pct=0.455858 moved_to_replicas=13521 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] avg=8192.0000 max=10404 max_rank=7 min=3449 min_rank=3 imbalance_ratio=1.270020 max_load_reduction=8716 max_load_reduction_pct=0.455858 moved_to_replicas=13521 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] avg=8192.0000 max=9525 max_rank=0 min=5896 min_rank=7 imbalance_ratio=1.162720 max_load_reduction=3654 max_load_reduction_pct=0.277259 moved_to_replicas=20495 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] avg=8192.0000 max=10404 max_rank=7 min=3449 min_rank=3 imbalance_ratio=1.270020 max_load_reduction=8716 max_load_reduction_pct=0.455858 moved_to_replicas=13521 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=14848 expected_recv=17013 expected_overflow=0 exact_recv_per_rank=[8376, 8481, 8480, 8479, 7995, 6765, 8480, 8480] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=14848 expected_recv=17013 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 1685, 3449, 19120, 2980, 9656, 12376] avg=8192.0000 max=19120 max_rank=4 min=1685 min_rank=2 imbalance_ratio=2.333984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=17152 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 1685, 3449, 19120, 2980, 9656, 12376] avg=8192.0000 max=19120 max_rank=4 min=1685 min_rank=2 imbalance_ratio=2.333984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[14260, 5599, 16667, 6743, 8738, 1707, 952, 10870] avg=8192.0000 max=16667 max_rank=2 min=952 min_rank=6 imbalance_ratio=2.034546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[16321, 1552, 3420, 3458, 1222, 10792, 10183, 18588] avg=8192.0000 max=18588 max_rank=7 min=1222 min_rank=4 imbalance_ratio=2.269043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[16321, 1552, 3420, 3458, 1222, 10792, 10183, 18588] avg=8192.0000 max=18588 max_rank=7 min=1222 min_rank=4 imbalance_ratio=2.269043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[16321, 1552, 3420, 3458, 1222, 10792, 10183, 18588] avg=8192.0000 max=18588 max_rank=7 min=1222 min_rank=4 imbalance_ratio=2.269043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 1685, 3449, 19120, 2980, 9656, 12376] avg=8192.0000 max=19120 max_rank=4 min=1685 min_rank=2 imbalance_ratio=2.333984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] avg=8192.0000 max=10404 max_rank=7 min=3449 min_rank=3 imbalance_ratio=1.270020 max_load_reduction=8716 max_load_reduction_pct=0.455858 moved_to_replicas=13521 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] avg=8192.0000 max=10404 max_rank=7 min=3449 min_rank=3 imbalance_ratio=1.270020 max_load_reduction=8716 max_load_reduction_pct=0.455858 moved_to_replicas=13521 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[10869, 5599, 10870, 6743, 8738, 5098, 6749, 10870] avg=8192.0000 max=10870 max_rank=2 min=5098 min_rank=5 imbalance_ratio=1.326904 max_load_reduction=5797 max_load_reduction_pct=0.347813 moved_to_replicas=21424 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[10791, 7082, 3420, 3458, 9018, 10792, 10183, 10792] avg=8192.0000 max=10792 max_rank=5 min=3420 min_rank=2 imbalance_ratio=1.317383 max_load_reduction=7796 max_load_reduction_pct=0.419410 moved_to_replicas=32770 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[10791, 7082, 3420, 3458, 9018, 10792, 10183, 10792] avg=8192.0000 max=10792 max_rank=5 min=3420 min_rank=2 imbalance_ratio=1.317383 max_load_reduction=7796 max_load_reduction_pct=0.419410 moved_to_replicas=32770 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[10791, 7082, 3420, 3458, 9018, 10792, 10183, 10792] avg=8192.0000 max=10792 max_rank=5 min=3420 min_rank=2 imbalance_ratio=1.317383 max_load_reduction=7796 max_load_reduction_pct=0.419410 moved_to_replicas=32770 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] avg=8192.0000 max=10404 max_rank=7 min=3449 min_rank=3 imbalance_ratio=1.270020 max_load_reduction=8716 max_load_reduction_pct=0.455858 moved_to_replicas=13521 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[16321, 1552, 3420, 3458, 1222, 10792, 10183, 18588] avg=8192.0000 max=18588 max_rank=7 min=1222 min_rank=4 imbalance_ratio=2.269043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[991, 24884, 5241, 2569, 13382, 3150, 7068, 8251] avg=8192.0000 max=24884 max_rank=1 min=991 min_rank=0 imbalance_ratio=3.037598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[16321, 1552, 3420, 3458, 1222, 10792, 10183, 18588] avg=8192.0000 max=18588 max_rank=7 min=1222 min_rank=4 imbalance_ratio=2.269043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[7452, 3076, 7886, 15854, 6651, 1860, 7472, 15285] avg=8192.0000 max=15854 max_rank=3 min=1860 min_rank=5 imbalance_ratio=1.935303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[14260, 5599, 16667, 6743, 8738, 1707, 952, 10870] avg=8192.0000 max=16667 max_rank=2 min=952 min_rank=6 imbalance_ratio=2.034546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[14260, 5599, 16667, 6743, 8738, 1707, 952, 10870] avg=8192.0000 max=16667 max_rank=2 min=952 min_rank=6 imbalance_ratio=2.034546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[14260, 5599, 16667, 6743, 8738, 1707, 952, 10870] avg=8192.0000 max=16667 max_rank=2 min=952 min_rank=6 imbalance_ratio=2.034546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[16321, 1552, 3420, 3458, 1222, 10792, 10183, 18588] avg=8192.0000 max=18588 max_rank=7 min=1222 min_rank=4 imbalance_ratio=2.269043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[10791, 7082, 3420, 3458, 9018, 10792, 10183, 10792] avg=8192.0000 max=10792 max_rank=5 min=3420 min_rank=2 imbalance_ratio=1.317383 max_load_reduction=7796 max_load_reduction_pct=0.419410 moved_to_replicas=32770 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[10791, 7082, 3420, 3458, 9018, 10792, 10183, 10792] avg=8192.0000 max=10792 max_rank=5 min=3420 min_rank=2 imbalance_ratio=1.317383 max_load_reduction=7796 max_load_reduction_pct=0.419410 moved_to_replicas=32770 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[7452, 9179, 7886, 9181, 6651, 8533, 7472, 9182] avg=8192.0000 max=9182 max_rank=7 min=6651 min_rank=4 imbalance_ratio=1.120850 max_load_reduction=6672 max_load_reduction_pct=0.420840 moved_to_replicas=18440 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[8522, 8523, 8524, 8524, 8522, 6025, 8523, 8373] avg=8192.0000 max=8524 max_rank=2 min=6025 min_rank=5 imbalance_ratio=1.040527 max_load_reduction=16360 max_load_reduction_pct=0.657451 moved_to_replicas=41587 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[10869, 5599, 10870, 6743, 8738, 5098, 6749, 10870] avg=8192.0000 max=10870 max_rank=2 min=5098 min_rank=5 imbalance_ratio=1.326904 max_load_reduction=5797 max_load_reduction_pct=0.347813 moved_to_replicas=21424 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[10869, 5599, 10870, 6743, 8738, 5098, 6749, 10870] avg=8192.0000 max=10870 max_rank=2 min=5098 min_rank=5 imbalance_ratio=1.326904 max_load_reduction=5797 max_load_reduction_pct=0.347813 moved_to_replicas=21424 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[10869, 5599, 10870, 6743, 8738, 5098, 6749, 10870] avg=8192.0000 max=10870 max_rank=2 min=5098 min_rank=5 imbalance_ratio=1.326904 max_load_reduction=5797 max_load_reduction_pct=0.347813 moved_to_replicas=21424 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[10791, 7082, 3420, 3458, 9018, 10792, 10183, 10792] avg=8192.0000 max=10792 max_rank=5 min=3420 min_rank=2 imbalance_ratio=1.317383 max_load_reduction=7796 max_load_reduction_pct=0.419410 moved_to_replicas=32770 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[14260, 5599, 16667, 6743, 8738, 1707, 952, 10870] avg=8192.0000 max=16667 max_rank=2 min=952 min_rank=6 imbalance_ratio=2.034546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[16250, 21022, 933, 1367, 5038, 3573, 9954, 7399] avg=8192.0000 max=21022 max_rank=1 min=933 min_rank=2 imbalance_ratio=2.566162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[14260, 5599, 16667, 6743, 8738, 1707, 952, 10870] avg=8192.0000 max=16667 max_rank=2 min=952 min_rank=6 imbalance_ratio=2.034546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[7452, 3076, 7886, 15854, 6651, 1860, 7472, 15285] avg=8192.0000 max=15854 max_rank=3 min=1860 min_rank=5 imbalance_ratio=1.935303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[7452, 3076, 7886, 15854, 6651, 1860, 7472, 15285] avg=8192.0000 max=15854 max_rank=3 min=1860 min_rank=5 imbalance_ratio=1.935303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[8387, 1691, 4070, 7203, 16186, 4440, 11688, 11871] avg=8192.0000 max=16186 max_rank=4 min=1691 min_rank=1 imbalance_ratio=1.975830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[7452, 3076, 7886, 15854, 6651, 1860, 7472, 15285] avg=8192.0000 max=15854 max_rank=3 min=1860 min_rank=5 imbalance_ratio=1.935303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[10869, 5599, 10870, 6743, 8738, 5098, 6749, 10870] avg=8192.0000 max=10870 max_rank=2 min=5098 min_rank=5 imbalance_ratio=1.326904 max_load_reduction=5797 max_load_reduction_pct=0.347813 moved_to_replicas=21424 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[14260, 5599, 16667, 6743, 8738, 1707, 952, 10870] avg=8192.0000 max=16667 max_rank=2 min=952 min_rank=6 imbalance_ratio=2.034546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[8697, 8696, 8696, 5930, 7469, 8695, 8696, 8657] avg=8192.0000 max=8697 max_rank=0 min=5930 min_rank=3 imbalance_ratio=1.061646 max_load_reduction=12325 max_load_reduction_pct=0.586291 moved_to_replicas=28502 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[10869, 5599, 10870, 6743, 8738, 5098, 6749, 10870] avg=8192.0000 max=10870 max_rank=2 min=5098 min_rank=5 imbalance_ratio=1.326904 max_load_reduction=5797 max_load_reduction_pct=0.347813 moved_to_replicas=21424 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[7452, 9179, 7886, 9181, 6651, 8533, 7472, 9182] avg=8192.0000 max=9182 max_rank=7 min=6651 min_rank=4 imbalance_ratio=1.120850 max_load_reduction=6672 max_load_reduction_pct=0.420840 moved_to_replicas=18440 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[7452, 9179, 7886, 9181, 6651, 8533, 7472, 9182] avg=8192.0000 max=9182 max_rank=7 min=6651 min_rank=4 imbalance_ratio=1.120850 max_load_reduction=6672 max_load_reduction_pct=0.420840 moved_to_replicas=18440 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[7452, 9179, 7886, 9181, 6651, 8533, 7472, 9182] avg=8192.0000 max=9182 max_rank=7 min=6651 min_rank=4 imbalance_ratio=1.120850 max_load_reduction=6672 max_load_reduction_pct=0.420840 moved_to_replicas=18440 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[8387, 8938, 7003, 7203, 8939, 7189, 8939, 8938] avg=8192.0000 max=8939 max_rank=4 min=7003 min_rank=2 imbalance_ratio=1.091187 max_load_reduction=7247 max_load_reduction_pct=0.447733 moved_to_replicas=22777 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[10869, 5599, 10870, 6743, 8738, 5098, 6749, 10870] avg=8192.0000 max=10870 max_rank=2 min=5098 min_rank=5 imbalance_ratio=1.326904 max_load_reduction=5797 max_load_reduction_pct=0.347813 moved_to_replicas=21424 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[7452, 3076, 7886, 15854, 6651, 1860, 7472, 15285] avg=8192.0000 max=15854 max_rank=3 min=1860 min_rank=5 imbalance_ratio=1.935303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[22401, 7978, 4287, 7164, 1135, 2753, 1985, 17833] avg=8192.0000 max=22401 max_rank=0 min=1135 min_rank=4 imbalance_ratio=2.734497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[7452, 3076, 7886, 15854, 6651, 1860, 7472, 15285] avg=8192.0000 max=15854 max_rank=3 min=1860 min_rank=5 imbalance_ratio=1.935303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[16250, 21022, 933, 1367, 5038, 3573, 9954, 7399] avg=8192.0000 max=21022 max_rank=1 min=933 min_rank=2 imbalance_ratio=2.566162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[16250, 21022, 933, 1367, 5038, 3573, 9954, 7399] avg=8192.0000 max=21022 max_rank=1 min=933 min_rank=2 imbalance_ratio=2.566162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[16250, 21022, 933, 1367, 5038, 3573, 9954, 7399] avg=8192.0000 max=21022 max_rank=1 min=933 min_rank=2 imbalance_ratio=2.566162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[8615, 1037, 7593, 20099, 902, 15779, 9023, 2488] avg=8192.0000 max=20099 max_rank=3 min=902 min_rank=4 imbalance_ratio=2.453491 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[7452, 9179, 7886, 9181, 6651, 8533, 7472, 9182] avg=8192.0000 max=9182 max_rank=7 min=6651 min_rank=4 imbalance_ratio=1.120850 max_load_reduction=6672 max_load_reduction_pct=0.420840 moved_to_replicas=18440 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[8724, 7978, 7424, 7164, 8724, 8725, 8073, 8724] avg=8192.0000 max=8725 max_rank=5 min=7164 min_rank=3 imbalance_ratio=1.065063 max_load_reduction=13676 max_load_reduction_pct=0.610508 moved_to_replicas=38567 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[7452, 3076, 7886, 15854, 6651, 1860, 7472, 15285] avg=8192.0000 max=15854 max_rank=3 min=1860 min_rank=5 imbalance_ratio=1.935303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[7452, 9179, 7886, 9181, 6651, 8533, 7472, 9182] avg=8192.0000 max=9182 max_rank=7 min=6651 min_rank=4 imbalance_ratio=1.120850 max_load_reduction=6672 max_load_reduction_pct=0.420840 moved_to_replicas=18440 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[8697, 8696, 8696, 5930, 7469, 8695, 8696, 8657] avg=8192.0000 max=8697 max_rank=0 min=5930 min_rank=3 imbalance_ratio=1.061646 max_load_reduction=12325 max_load_reduction_pct=0.586291 moved_to_replicas=28502 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[8697, 8696, 8696, 5930, 7469, 8695, 8696, 8657] avg=8192.0000 max=8697 max_rank=0 min=5930 min_rank=3 imbalance_ratio=1.061646 max_load_reduction=12325 max_load_reduction_pct=0.586291 moved_to_replicas=28502 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[8697, 8696, 8696, 5930, 7469, 8695, 8696, 8657] avg=8192.0000 max=8697 max_rank=0 min=5930 min_rank=3 imbalance_ratio=1.061646 max_load_reduction=12325 max_load_reduction_pct=0.586291 moved_to_replicas=28502 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[8615, 3770, 7593, 9133, 9135, 9134, 9023, 9133] avg=8192.0000 max=9135 max_rank=4 min=3770 min_rank=1 imbalance_ratio=1.115112 max_load_reduction=10964 max_load_reduction_pct=0.545500 moved_to_replicas=33906 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[7452, 9179, 7886, 9181, 6651, 8533, 7472, 9182] avg=8192.0000 max=9182 max_rank=7 min=6651 min_rank=4 imbalance_ratio=1.120850 max_load_reduction=6672 max_load_reduction_pct=0.420840 moved_to_replicas=18440 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[16250, 21022, 933, 1367, 5038, 3573, 9954, 7399] avg=8192.0000 max=21022 max_rank=1 min=933 min_rank=2 imbalance_ratio=2.566162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[9557, 5725, 3495, 20515, 6046, 11422, 7712, 1064] avg=8192.0000 max=20515 max_rank=3 min=1064 min_rank=7 imbalance_ratio=2.504272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[16250, 21022, 933, 1367, 5038, 3573, 9954, 7399] avg=8192.0000 max=21022 max_rank=1 min=933 min_rank=2 imbalance_ratio=2.566162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[22401, 7978, 4287, 7164, 1135, 2753, 1985, 17833] avg=8192.0000 max=22401 max_rank=0 min=1135 min_rank=4 imbalance_ratio=2.734497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[22401, 7978, 4287, 7164, 1135, 2753, 1985, 17833] avg=8192.0000 max=22401 max_rank=0 min=1135 min_rank=4 imbalance_ratio=2.734497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[22401, 7978, 4287, 7164, 1135, 2753, 1985, 17833] avg=8192.0000 max=22401 max_rank=0 min=1135 min_rank=4 imbalance_ratio=2.734497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[5823, 20298, 4359, 6463, 10601, 2673, 9588, 5731] avg=8192.0000 max=20298 max_rank=1 min=2673 min_rank=5 imbalance_ratio=2.477783 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[16250, 21022, 933, 1367, 5038, 3573, 9954, 7399] avg=8192.0000 max=21022 max_rank=1 min=933 min_rank=2 imbalance_ratio=2.566162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[8697, 8696, 8696, 5930, 7469, 8695, 8696, 8657] avg=8192.0000 max=8697 max_rank=0 min=5930 min_rank=3 imbalance_ratio=1.061646 max_load_reduction=12325 max_load_reduction_pct=0.586291 moved_to_replicas=28502 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[8686, 8461, 7702, 8685, 6917, 8686, 7712, 8687] avg=8192.0000 max=8687 max_rank=7 min=6917 min_rank=4 imbalance_ratio=1.060425 max_load_reduction=11828 max_load_reduction_pct=0.576554 moved_to_replicas=34045 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[8697, 8696, 8696, 5930, 7469, 8695, 8696, 8657] avg=8192.0000 max=8697 max_rank=0 min=5930 min_rank=3 imbalance_ratio=1.061646 max_load_reduction=12325 max_load_reduction_pct=0.586291 moved_to_replicas=28502 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[8724, 7978, 7424, 7164, 8724, 8725, 8073, 8724] avg=8192.0000 max=8725 max_rank=5 min=7164 min_rank=3 imbalance_ratio=1.065063 max_load_reduction=13676 max_load_reduction_pct=0.610508 moved_to_replicas=38567 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[8724, 7978, 7424, 7164, 8724, 8725, 8073, 8724] avg=8192.0000 max=8725 max_rank=5 min=7164 min_rank=3 imbalance_ratio=1.065063 max_load_reduction=13676 max_load_reduction_pct=0.610508 moved_to_replicas=38567 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[8724, 7978, 7424, 7164, 8724, 8725, 8073, 8724] avg=8192.0000 max=8725 max_rank=5 min=7164 min_rank=3 imbalance_ratio=1.065063 max_load_reduction=13676 max_load_reduction_pct=0.610508 moved_to_replicas=38567 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[8150, 8273, 8274, 7777, 8274, 8275, 8274, 8239] avg=8192.0000 max=8275 max_rank=5 min=7777 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=12023 max_load_reduction_pct=0.592324 moved_to_replicas=30131 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[8697, 8696, 8696, 5930, 7469, 8695, 8696, 8657] avg=8192.0000 max=8697 max_rank=0 min=5930 min_rank=3 imbalance_ratio=1.061646 max_load_reduction=12325 max_load_reduction_pct=0.586291 moved_to_replicas=28502 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[22401, 7978, 4287, 7164, 1135, 2753, 1985, 17833] avg=8192.0000 max=22401 max_rank=0 min=1135 min_rank=4 imbalance_ratio=2.734497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[22401, 7978, 4287, 7164, 1135, 2753, 1985, 17833] avg=8192.0000 max=22401 max_rank=0 min=1135 min_rank=4 imbalance_ratio=2.734497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[9557, 5725, 3495, 20515, 6046, 11422, 7712, 1064] avg=8192.0000 max=20515 max_rank=3 min=1064 min_rank=7 imbalance_ratio=2.504272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[14061, 5050, 1629, 20481, 5647, 4245, 1147, 13276] avg=8192.0000 max=20481 max_rank=3 min=1147 min_rank=6 imbalance_ratio=2.500122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[9557, 5725, 3495, 20515, 6046, 11422, 7712, 1064] avg=8192.0000 max=20515 max_rank=3 min=1064 min_rank=7 imbalance_ratio=2.504272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[9557, 5725, 3495, 20515, 6046, 11422, 7712, 1064] avg=8192.0000 max=20515 max_rank=3 min=1064 min_rank=7 imbalance_ratio=2.504272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[8724, 7978, 7424, 7164, 8724, 8725, 8073, 8724] avg=8192.0000 max=8725 max_rank=5 min=7164 min_rank=3 imbalance_ratio=1.065063 max_load_reduction=13676 max_load_reduction_pct=0.610508 moved_to_replicas=38567 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[8724, 7978, 7424, 7164, 8724, 8725, 8073, 8724] avg=8192.0000 max=8725 max_rank=5 min=7164 min_rank=3 imbalance_ratio=1.065063 max_load_reduction=13676 max_load_reduction_pct=0.610508 moved_to_replicas=38567 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[8686, 8461, 7702, 8685, 6917, 8686, 7712, 8687] avg=8192.0000 max=8687 max_rank=7 min=6917 min_rank=4 imbalance_ratio=1.060425 max_load_reduction=11828 max_load_reduction_pct=0.576554 moved_to_replicas=34045 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[14061, 5050, 1629, 14061, 5647, 4245, 7567, 13276] avg=8192.0000 max=14061 max_rank=0 min=1629 min_rank=2 imbalance_ratio=1.716431 max_load_reduction=6420 max_load_reduction_pct=0.313461 moved_to_replicas=17592 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[8686, 8461, 7702, 8685, 6917, 8686, 7712, 8687] avg=8192.0000 max=8687 max_rank=7 min=6917 min_rank=4 imbalance_ratio=1.060425 max_load_reduction=11828 max_load_reduction_pct=0.576554 moved_to_replicas=34045 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[22401, 7978, 4287, 7164, 1135, 2753, 1985, 17833] avg=8192.0000 max=22401 max_rank=0 min=1135 min_rank=4 imbalance_ratio=2.734497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[8686, 8461, 7702, 8685, 6917, 8686, 7712, 8687] avg=8192.0000 max=8687 max_rank=7 min=6917 min_rank=4 imbalance_ratio=1.060425 max_load_reduction=11828 max_load_reduction_pct=0.576554 moved_to_replicas=34045 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[12437, 13179, 4062, 7594, 7742, 9524, 5102, 5896] avg=8192.0000 max=13179 max_rank=1 min=4062 min_rank=2 imbalance_ratio=1.608765 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[9557, 5725, 3495, 20515, 6046, 11422, 7712, 1064] avg=8192.0000 max=20515 max_rank=3 min=1064 min_rank=7 imbalance_ratio=2.504272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[8724, 7978, 7424, 7164, 8724, 8725, 8073, 8724] avg=8192.0000 max=8725 max_rank=5 min=7164 min_rank=3 imbalance_ratio=1.065063 max_load_reduction=13676 max_load_reduction_pct=0.610508 moved_to_replicas=38567 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[9557, 5725, 3495, 20515, 6046, 11422, 7712, 1064] avg=8192.0000 max=20515 max_rank=3 min=1064 min_rank=7 imbalance_ratio=2.504272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[14061, 5050, 1629, 20481, 5647, 4245, 1147, 13276] avg=8192.0000 max=20481 max_rank=3 min=1147 min_rank=6 imbalance_ratio=2.500122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[9525, 9524, 7717, 7594, 7742, 9524, 8014, 5896] avg=8192.0000 max=9525 max_rank=0 min=5896 min_rank=7 imbalance_ratio=1.162720 max_load_reduction=3654 max_load_reduction_pct=0.277259 moved_to_replicas=20495 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[14061, 5050, 1629, 20481, 5647, 4245, 1147, 13276] avg=8192.0000 max=20481 max_rank=3 min=1147 min_rank=6 imbalance_ratio=2.500122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[3795, 4820, 12058, 17999, 10160, 7423, 1432, 7849] avg=8192.0000 max=17999 max_rank=3 min=1432 min_rank=6 imbalance_ratio=2.197144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[14061, 5050, 1629, 20481, 5647, 4245, 1147, 13276] avg=8192.0000 max=20481 max_rank=3 min=1147 min_rank=6 imbalance_ratio=2.500122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[8686, 8461, 7702, 8685, 6917, 8686, 7712, 8687] avg=8192.0000 max=8687 max_rank=7 min=6917 min_rank=4 imbalance_ratio=1.060425 max_load_reduction=11828 max_load_reduction_pct=0.576554 moved_to_replicas=34045 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[8686, 8461, 7702, 8685, 6917, 8686, 7712, 8687] avg=8192.0000 max=8687 max_rank=7 min=6917 min_rank=4 imbalance_ratio=1.060425 max_load_reduction=11828 max_load_reduction_pct=0.576554 moved_to_replicas=34045 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[14061, 5050, 1629, 14061, 5647, 4245, 7567, 13276] avg=8192.0000 max=14061 max_rank=0 min=1629 min_rank=2 imbalance_ratio=1.716431 max_load_reduction=6420 max_load_reduction_pct=0.313461 moved_to_replicas=17592 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[14061, 5050, 1629, 14061, 5647, 4245, 7567, 13276] avg=8192.0000 max=14061 max_rank=0 min=1629 min_rank=2 imbalance_ratio=1.716431 max_load_reduction=6420 max_load_reduction_pct=0.313461 moved_to_replicas=17592 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[5958, 8244, 8634, 8634, 8635, 8633, 8634, 8164] avg=8192.0000 max=8635 max_rank=4 min=5958 min_rank=0 imbalance_ratio=1.054077 max_load_reduction=9364 max_load_reduction_pct=0.520251 moved_to_replicas=27225 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[14061, 5050, 1629, 14061, 5647, 4245, 7567, 13276] avg=8192.0000 max=14061 max_rank=0 min=1629 min_rank=2 imbalance_ratio=1.716431 max_load_reduction=6420 max_load_reduction_pct=0.313461 moved_to_replicas=17592 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[9557, 5725, 3495, 20515, 6046, 11422, 7712, 1064] avg=8192.0000 max=20515 max_rank=3 min=1064 min_rank=7 imbalance_ratio=2.504272 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 1685, 3449, 19120, 2980, 9656, 12376] avg=8192.0000 max=19120 max_rank=4 min=1685 min_rank=2 imbalance_ratio=2.333984 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[14061, 5050, 1629, 20481, 5647, 4245, 1147, 13276] avg=8192.0000 max=20481 max_rank=3 min=1147 min_rank=6 imbalance_ratio=2.500122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[14061, 5050, 1629, 20481, 5647, 4245, 1147, 13276] avg=8192.0000 max=20481 max_rank=3 min=1147 min_rank=6 imbalance_ratio=2.500122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[8686, 8461, 7702, 8685, 6917, 8686, 7712, 8687] avg=8192.0000 max=8687 max_rank=7 min=6917 min_rank=4 imbalance_ratio=1.060425 max_load_reduction=11828 max_load_reduction_pct=0.576554 moved_to_replicas=34045 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[3795, 4820, 12058, 17999, 10160, 7423, 1432, 7849] avg=8192.0000 max=17999 max_rank=3 min=1432 min_rank=6 imbalance_ratio=2.197144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[9267, 7003, 10403, 3449, 10402, 4952, 9656, 10404] avg=8192.0000 max=10404 max_rank=7 min=3449 min_rank=3 imbalance_ratio=1.270020 max_load_reduction=8716 max_load_reduction_pct=0.455858 moved_to_replicas=13521 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[3795, 4820, 12058, 17999, 10160, 7423, 1432, 7849] avg=8192.0000 max=17999 max_rank=3 min=1432 min_rank=6 imbalance_ratio=2.197144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[548, 6995, 14884, 9224, 2344, 17504, 10896, 3141] avg=8192.0000 max=17504 max_rank=5 min=548 min_rank=0 imbalance_ratio=2.136719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[3795, 4820, 12058, 17999, 10160, 7423, 1432, 7849] avg=8192.0000 max=17999 max_rank=3 min=1432 min_rank=6 imbalance_ratio=2.197144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[14061, 5050, 1629, 14061, 5647, 4245, 7567, 13276] avg=8192.0000 max=14061 max_rank=0 min=1629 min_rank=2 imbalance_ratio=1.716431 max_load_reduction=6420 max_load_reduction_pct=0.313461 moved_to_replicas=17592 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[14061, 5050, 1629, 14061, 5647, 4245, 7567, 13276] avg=8192.0000 max=14061 max_rank=0 min=1629 min_rank=2 imbalance_ratio=1.716431 max_load_reduction=6420 max_load_reduction_pct=0.313461 moved_to_replicas=17592 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[5958, 8244, 8634, 8634, 8635, 8633, 8634, 8164] avg=8192.0000 max=8635 max_rank=4 min=5958 min_rank=0 imbalance_ratio=1.054077 max_load_reduction=9364 max_load_reduction_pct=0.520251 moved_to_replicas=27225 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[5958, 8244, 8634, 8634, 8635, 8633, 8634, 8164] avg=8192.0000 max=8635 max_rank=4 min=5958 min_rank=0 imbalance_ratio=1.054077 max_load_reduction=9364 max_load_reduction_pct=0.520251 moved_to_replicas=27225 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[14061, 5050, 1629, 20481, 5647, 4245, 1147, 13276] avg=8192.0000 max=20481 max_rank=3 min=1147 min_rank=6 imbalance_ratio=2.500122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[5958, 8244, 8634, 8634, 8635, 8633, 8634, 8164] avg=8192.0000 max=8635 max_rank=4 min=5958 min_rank=0 imbalance_ratio=1.054077 max_load_reduction=9364 max_load_reduction_pct=0.520251 moved_to_replicas=27225 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[7156, 6995, 10896, 9224, 6332, 10896, 10896, 3141] avg=8192.0000 max=10896 max_rank=2 min=3141 min_rank=7 imbalance_ratio=1.330078 max_load_reduction=6608 max_load_reduction_pct=0.377514 moved_to_replicas=27097 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[16321, 1552, 3420, 3458, 1222, 10792, 10183, 18588] avg=8192.0000 max=18588 max_rank=7 min=1222 min_rank=4 imbalance_ratio=2.269043 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[3795, 4820, 12058, 17999, 10160, 7423, 1432, 7849] avg=8192.0000 max=17999 max_rank=3 min=1432 min_rank=6 imbalance_ratio=2.197144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[3795, 4820, 12058, 17999, 10160, 7423, 1432, 7849] avg=8192.0000 max=17999 max_rank=3 min=1432 min_rank=6 imbalance_ratio=2.197144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[14061, 5050, 1629, 14061, 5647, 4245, 7567, 13276] avg=8192.0000 max=14061 max_rank=0 min=1629 min_rank=2 imbalance_ratio=1.716431 max_load_reduction=6420 max_load_reduction_pct=0.313461 moved_to_replicas=17592 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[548, 6995, 14884, 9224, 2344, 17504, 10896, 3141] avg=8192.0000 max=17504 max_rank=5 min=548 min_rank=0 imbalance_ratio=2.136719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[548, 6995, 14884, 9224, 2344, 17504, 10896, 3141] avg=8192.0000 max=17504 max_rank=5 min=548 min_rank=0 imbalance_ratio=2.136719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[10791, 7082, 3420, 3458, 9018, 10792, 10183, 10792] avg=8192.0000 max=10792 max_rank=5 min=3420 min_rank=2 imbalance_ratio=1.317383 max_load_reduction=7796 max_load_reduction_pct=0.419410 moved_to_replicas=32770 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[548, 6995, 14884, 9224, 2344, 17504, 10896, 3141] avg=8192.0000 max=17504 max_rank=5 min=548 min_rank=0 imbalance_ratio=2.136719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[3766, 20542, 5497, 1028, 6307, 6765, 5648, 15983] avg=8192.0000 max=20542 max_rank=1 min=1028 min_rank=3 imbalance_ratio=2.507568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[5958, 8244, 8634, 8634, 8635, 8633, 8634, 8164] avg=8192.0000 max=8635 max_rank=4 min=5958 min_rank=0 imbalance_ratio=1.054077 max_load_reduction=9364 max_load_reduction_pct=0.520251 moved_to_replicas=27225 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[5958, 8244, 8634, 8634, 8635, 8633, 8634, 8164] avg=8192.0000 max=8635 max_rank=4 min=5958 min_rank=0 imbalance_ratio=1.054077 max_load_reduction=9364 max_load_reduction_pct=0.520251 moved_to_replicas=27225 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[7156, 6995, 10896, 9224, 6332, 10896, 10896, 3141] avg=8192.0000 max=10896 max_rank=2 min=3141 min_rank=7 imbalance_ratio=1.330078 max_load_reduction=6608 max_load_reduction_pct=0.377514 moved_to_replicas=27097 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[7156, 6995, 10896, 9224, 6332, 10896, 10896, 3141] avg=8192.0000 max=10896 max_rank=2 min=3141 min_rank=7 imbalance_ratio=1.330078 max_load_reduction=6608 max_load_reduction_pct=0.377514 moved_to_replicas=27097 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[7156, 6995, 10896, 9224, 6332, 10896, 10896, 3141] avg=8192.0000 max=10896 max_rank=2 min=3141 min_rank=7 imbalance_ratio=1.330078 max_load_reduction=6608 max_load_reduction_pct=0.377514 moved_to_replicas=27097 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[8376, 8481, 8480, 8479, 7995, 6765, 8480, 8480] avg=8192.0000 max=8481 max_rank=1 min=6765 min_rank=5 imbalance_ratio=1.035278 max_load_reduction=12061 max_load_reduction_pct=0.587139 moved_to_replicas=29880 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[3795, 4820, 12058, 17999, 10160, 7423, 1432, 7849] avg=8192.0000 max=17999 max_rank=3 min=1432 min_rank=6 imbalance_ratio=2.197144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[14260, 5599, 16667, 6743, 8738, 1707, 952, 10870] avg=8192.0000 max=16667 max_rank=2 min=952 min_rank=6 imbalance_ratio=2.034546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[548, 6995, 14884, 9224, 2344, 17504, 10896, 3141] avg=8192.0000 max=17504 max_rank=5 min=548 min_rank=0 imbalance_ratio=2.136719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[548, 6995, 14884, 9224, 2344, 17504, 10896, 3141] avg=8192.0000 max=17504 max_rank=5 min=548 min_rank=0 imbalance_ratio=2.136719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[3766, 20542, 5497, 1028, 6307, 6765, 5648, 15983] avg=8192.0000 max=20542 max_rank=1 min=1028 min_rank=3 imbalance_ratio=2.507568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[5958, 8244, 8634, 8634, 8635, 8633, 8634, 8164] avg=8192.0000 max=8635 max_rank=4 min=5958 min_rank=0 imbalance_ratio=1.054077 max_load_reduction=9364 max_load_reduction_pct=0.520251 moved_to_replicas=27225 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[3766, 20542, 5497, 1028, 6307, 6765, 5648, 15983] avg=8192.0000 max=20542 max_rank=1 min=1028 min_rank=3 imbalance_ratio=2.507568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[3766, 20542, 5497, 1028, 6307, 6765, 5648, 15983] avg=8192.0000 max=20542 max_rank=1 min=1028 min_rank=3 imbalance_ratio=2.507568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[10869, 5599, 10870, 6743, 8738, 5098, 6749, 10870] avg=8192.0000 max=10870 max_rank=2 min=5098 min_rank=5 imbalance_ratio=1.326904 max_load_reduction=5797 max_load_reduction_pct=0.347813 moved_to_replicas=21424 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[7156, 6995, 10896, 9224, 6332, 10896, 10896, 3141] avg=8192.0000 max=10896 max_rank=2 min=3141 min_rank=7 imbalance_ratio=1.330078 max_load_reduction=6608 max_load_reduction_pct=0.377514 moved_to_replicas=27097 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[7156, 6995, 10896, 9224, 6332, 10896, 10896, 3141] avg=8192.0000 max=10896 max_rank=2 min=3141 min_rank=7 imbalance_ratio=1.330078 max_load_reduction=6608 max_load_reduction_pct=0.377514 moved_to_replicas=27097 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[8376, 8481, 8480, 8479, 7995, 6765, 8480, 8480] avg=8192.0000 max=8481 max_rank=1 min=6765 min_rank=5 imbalance_ratio=1.035278 max_load_reduction=12061 max_load_reduction_pct=0.587139 moved_to_replicas=29880 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[8376, 8481, 8480, 8479, 7995, 6765, 8480, 8480] avg=8192.0000 max=8481 max_rank=1 min=6765 min_rank=5 imbalance_ratio=1.035278 max_load_reduction=12061 max_load_reduction_pct=0.587139 moved_to_replicas=29880 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[8376, 8481, 8480, 8479, 7995, 6765, 8480, 8480] avg=8192.0000 max=8481 max_rank=1 min=6765 min_rank=5 imbalance_ratio=1.035278 max_load_reduction=12061 max_load_reduction_pct=0.587139 moved_to_replicas=29880 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[548, 6995, 14884, 9224, 2344, 17504, 10896, 3141] avg=8192.0000 max=17504 max_rank=5 min=548 min_rank=0 imbalance_ratio=2.136719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[7452, 3076, 7886, 15854, 6651, 1860, 7472, 15285] avg=8192.0000 max=15854 max_rank=3 min=1860 min_rank=5 imbalance_ratio=1.935303 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[3766, 20542, 5497, 1028, 6307, 6765, 5648, 15983] avg=8192.0000 max=20542 max_rank=1 min=1028 min_rank=3 imbalance_ratio=2.507568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[3766, 20542, 5497, 1028, 6307, 6765, 5648, 15983] avg=8192.0000 max=20542 max_rank=1 min=1028 min_rank=3 imbalance_ratio=2.507568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[7156, 6995, 10896, 9224, 6332, 10896, 10896, 3141] avg=8192.0000 max=10896 max_rank=2 min=3141 min_rank=7 imbalance_ratio=1.330078 max_load_reduction=6608 max_load_reduction_pct=0.377514 moved_to_replicas=27097 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[7452, 9179, 7886, 9181, 6651, 8533, 7472, 9182] avg=8192.0000 max=9182 max_rank=7 min=6651 min_rank=4 imbalance_ratio=1.120850 max_load_reduction=6672 max_load_reduction_pct=0.420840 moved_to_replicas=18440 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[8376, 8481, 8480, 8479, 7995, 6765, 8480, 8480] avg=8192.0000 max=8481 max_rank=1 min=6765 min_rank=5 imbalance_ratio=1.035278 max_load_reduction=12061 max_load_reduction_pct=0.587139 moved_to_replicas=29880 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[8376, 8481, 8480, 8479, 7995, 6765, 8480, 8480] avg=8192.0000 max=8481 max_rank=1 min=6765 min_rank=5 imbalance_ratio=1.035278 max_load_reduction=12061 max_load_reduction_pct=0.587139 moved_to_replicas=29880 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[3766, 20542, 5497, 1028, 6307, 6765, 5648, 15983] avg=8192.0000 max=20542 max_rank=1 min=1028 min_rank=3 imbalance_ratio=2.507568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[16250, 21022, 933, 1367, 5038, 3573, 9954, 7399] avg=8192.0000 max=21022 max_rank=1 min=933 min_rank=2 imbalance_ratio=2.566162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[8376, 8481, 8480, 8479, 7995, 6765, 8480, 8480] avg=8192.0000 max=8481 max_rank=1 min=6765 min_rank=5 imbalance_ratio=1.035278 max_load_reduction=12061 max_load_reduction_pct=0.587139 moved_to_replicas=29880 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[8697, 8696, 8696, 5930, 7469, 8695, 8696, 8657] avg=8192.0000 max=8697 max_rank=0 min=5930 min_rank=3 imbalance_ratio=1.061646 max_load_reduction=12325 max_load_reduction_pct=0.586291 moved_to_replicas=28502 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[22401, 7978, 4287, 7164, 1135, 2753, 1985, 17833] avg=8192.0000 max=22401 max_rank=0 min=1135 min_rank=4 imbalance_ratio=2.734497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[8724, 7978, 7424, 7164, 8724, 8725, 8073, 8724] avg=8192.0000 max=8725 max_rank=5 min=7164 min_rank=3 imbalance_ratio=1.065063 max_load_reduction=13676 max_load_reduction_pct=0.610508 moved_to_replicas=38567 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=284540 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=188160 peak_demand=14061 required_capacity=1.716431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[9557, 5725, 3495, 20515, 6046, 11422, 7712, 1064] avg=8192.0000 max=20515 max_rank=3 min=1064 min_rank=7 imbalance_ratio=2.504272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=32 total_expected_recv=255363 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=216704 peak_demand=10896 required_capacity=1.330078 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=32 total_expected_recv=277350 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=195072 peak_demand=13276 required_capacity=1.620605 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[8686, 8461, 7702, 8685, 6917, 8686, 7712, 8687] avg=8192.0000 max=8687 max_rank=7 min=6917 min_rank=4 imbalance_ratio=1.060425 max_load_reduction=11828 max_load_reduction_pct=0.576554 moved_to_replicas=34045 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=32 total_expected_recv=242239 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=229888 peak_demand=10896 required_capacity=1.330078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[14061, 5050, 1629, 20481, 5647, 4245, 1147, 13276] avg=8192.0000 max=20481 max_rank=3 min=1147 min_rank=6 imbalance_ratio=2.500122 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=32 total_expected_recv=258630 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=213504 peak_demand=11101 required_capacity=1.355103 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=32 total_expected_recv=262011 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=210304 peak_demand=11185 required_capacity=1.365356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[14061, 5050, 1629, 14061, 5647, 4245, 7567, 13276] avg=8192.0000 max=14061 max_rank=0 min=1629 min_rank=2 imbalance_ratio=1.716431 max_load_reduction=6420 max_load_reduction_pct=0.313461 moved_to_replicas=17592 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=32 total_expected_recv=259910 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=212608 peak_demand=14061 required_capacity=1.716431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[3795, 4820, 12058, 17999, 10160, 7423, 1432, 7849] avg=8192.0000 max=17999 max_rank=3 min=1432 min_rank=6 imbalance_ratio=2.197144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[5958, 8244, 8634, 8634, 8635, 8633, 8634, 8164] avg=8192.0000 max=8635 max_rank=4 min=5958 min_rank=0 imbalance_ratio=1.054077 max_load_reduction=9364 max_load_reduction_pct=0.520251 moved_to_replicas=27225 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[548, 6995, 14884, 9224, 2344, 17504, 10896, 3141] avg=8192.0000 max=17504 max_rank=5 min=548 min_rank=0 imbalance_ratio=2.136719 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[7156, 6995, 10896, 9224, 6332, 10896, 10896, 3141] avg=8192.0000 max=10896 max_rank=2 min=3141 min_rank=7 imbalance_ratio=1.330078 max_load_reduction=6608 max_load_reduction_pct=0.377514 moved_to_replicas=27097 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[3766, 20542, 5497, 1028, 6307, 6765, 5648, 15983] avg=8192.0000 max=20542 max_rank=1 min=1028 min_rank=3 imbalance_ratio=2.507568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[8376, 8481, 8480, 8479, 7995, 6765, 8480, 8480] avg=8192.0000 max=8481 max_rank=1 min=6765 min_rank=5 imbalance_ratio=1.035278 max_load_reduction=12061 max_load_reduction_pct=0.587139 moved_to_replicas=29880 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=32 total_expected_recv=257109 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=215168 peak_demand=11218 required_capacity=1.369385 + [2026-08-30 05:09:10] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 28220.4 | throughput per GPU (TFLOP/s/GPU): 11.8 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.104376E+01 | loss scale: 1.0 | grad norm: 35.708 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 13312 | min safe capacity: 1.6250 | +Number of parameters in transformer block in billions: 23.22 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 58333.29638671875 | max allocated: 79974.68603515625 | reserved: 84920.0 | max reserved: 84920.0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=14848 expected_recv=15914 expected_overflow=0 exact_recv_per_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=14848 expected_recv=17321 expected_overflow=0 exact_recv_per_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=14848 expected_recv=13288 expected_overflow=0 exact_recv_per_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=14848 expected_recv=17322 expected_overflow=0 exact_recv_per_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=14848 expected_recv=17323 expected_overflow=0 exact_recv_per_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=14848 expected_recv=15261 expected_overflow=0 exact_recv_per_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=14848 expected_recv=17322 expected_overflow=0 exact_recv_per_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=14848 expected_recv=15914 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=14848 expected_recv=17321 expected_overflow=0 exact_recv_per_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=14848 expected_recv=17321 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=14848 expected_recv=17322 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=14848 expected_recv=13288 actual_valid_recv=13568 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=14848 expected_recv=17323 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=14848 expected_recv=15261 actual_valid_recv=15616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17408 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17408 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=13568 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=14848 expected_recv=17322 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17536 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=14848 expected_recv=17321 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=15616 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17408 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17408 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=14848 expected_recv=8676 expected_overflow=0 exact_recv_per_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=14848 expected_recv=19603 expected_overflow=0 exact_recv_per_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=14848 expected_recv=8676 actual_valid_recv=8832 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=14848 expected_recv=11677 expected_overflow=0 exact_recv_per_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=14848 expected_recv=18812 expected_overflow=0 exact_recv_per_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=14848 expected_recv=15429 expected_overflow=0 exact_recv_per_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=8832 slack_slots=20864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=14848 expected_recv=19603 actual_valid_recv=19712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=14848 expected_recv=11677 actual_valid_recv=11904 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=14848 expected_recv=18812 actual_valid_recv=18944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=14848 expected_recv=15429 actual_valid_recv=15744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=19712 slack_slots=9984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=11904 slack_slots=17792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=14848 expected_recv=19930 expected_overflow=0 exact_recv_per_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=18944 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=15744 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=14848 expected_recv=19985 expected_overflow=0 exact_recv_per_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=14848 expected_recv=16960 expected_overflow=0 exact_recv_per_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=14848 expected_recv=19930 actual_valid_recv=19968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=14848 expected_recv=19985 actual_valid_recv=20096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=14848 expected_recv=17979 expected_overflow=0 exact_recv_per_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=19968 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=14848 expected_recv=16960 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=20096 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=14848 expected_recv=9243 expected_overflow=0 exact_recv_per_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=14848 expected_recv=17979 actual_valid_recv=18048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=17152 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=14848 expected_recv=19484 expected_overflow=0 exact_recv_per_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=14848 expected_recv=19484 expected_overflow=0 exact_recv_per_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=14848 expected_recv=9243 actual_valid_recv=9472 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=14848 expected_recv=12906 expected_overflow=0 exact_recv_per_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=18048 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=14848 expected_recv=19484 actual_valid_recv=19584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=14848 expected_recv=19484 actual_valid_recv=19584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=9472 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=14848 expected_recv=12906 actual_valid_recv=13312 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=19584 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=19584 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=13312 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=14848 expected_recv=18708 expected_overflow=0 exact_recv_per_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=14848 expected_recv=16475 expected_overflow=0 exact_recv_per_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=14848 expected_recv=18708 actual_valid_recv=18816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=14848 expected_recv=16793 expected_overflow=0 exact_recv_per_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=14848 expected_recv=16475 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=14848 expected_recv=17045 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=18816 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=14848 expected_recv=16793 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=14848 expected_recv=17047 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=14848 expected_recv=17045 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=14848 expected_recv=17047 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=17280 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=14848 expected_recv=16850 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=14848 expected_recv=15297 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=14848 expected_recv=17046 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=17280 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=14848 expected_recv=16850 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=14848 expected_recv=16131 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=14848 expected_recv=15297 actual_valid_recv=15616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=14848 expected_recv=17046 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=14848 expected_recv=17045 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=15616 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=14848 expected_recv=14611 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=14848 expected_recv=16131 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=17152 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=14848 expected_recv=17045 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=14848 expected_recv=16677 expected_overflow=0 exact_recv_per_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=14848 expected_recv=14611 actual_valid_recv=14720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=17280 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=14848 expected_recv=16677 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=14720 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=14848 expected_recv=16341 expected_overflow=0 exact_recv_per_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=14848 expected_recv=16341 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=14848 expected_recv=16388 expected_overflow=0 exact_recv_per_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=14848 expected_recv=16405 expected_overflow=0 exact_recv_per_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=14848 expected_recv=15397 expected_overflow=0 exact_recv_per_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=14848 expected_recv=16388 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=14848 expected_recv=16677 expected_overflow=0 exact_recv_per_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=14848 expected_recv=16405 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=14848 expected_recv=16510 expected_overflow=0 exact_recv_per_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=14848 expected_recv=15397 actual_valid_recv=15744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=14848 expected_recv=16677 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=14848 expected_recv=16677 expected_overflow=0 exact_recv_per_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=14848 expected_recv=16510 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=15744 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=14848 expected_recv=16915 expected_overflow=0 exact_recv_per_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=14848 expected_recv=16677 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=14848 expected_recv=16915 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=14848 expected_recv=16108 expected_overflow=0 exact_recv_per_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=17152 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=14848 expected_recv=16912 expected_overflow=0 exact_recv_per_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=14848 expected_recv=16108 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=14848 expected_recv=16262 expected_overflow=0 exact_recv_per_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=14848 expected_recv=16912 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=14848 expected_recv=16696 expected_overflow=0 exact_recv_per_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=14848 expected_recv=16911 expected_overflow=0 exact_recv_per_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=14848 expected_recv=16262 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=14848 expected_recv=16696 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=14848 expected_recv=16344 expected_overflow=0 exact_recv_per_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=14848 expected_recv=16911 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=14848 expected_recv=14924 expected_overflow=0 exact_recv_per_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=14848 expected_recv=18950 expected_overflow=0 exact_recv_per_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=14848 expected_recv=16344 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=14848 expected_recv=14924 actual_valid_recv=15232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=14848 expected_recv=18950 actual_valid_recv=19200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=14848 expected_recv=19243 expected_overflow=0 exact_recv_per_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=15232 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=19200 slack_slots=10496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=14848 expected_recv=16638 expected_overflow=0 exact_recv_per_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=14848 expected_recv=19243 actual_valid_recv=19456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=14848 expected_recv=7366 expected_overflow=0 exact_recv_per_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=14848 expected_recv=16638 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=19456 slack_slots=10240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=14848 expected_recv=14135 expected_overflow=0 exact_recv_per_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=14848 expected_recv=7366 actual_valid_recv=7552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=14848 expected_recv=16250 expected_overflow=0 exact_recv_per_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=14848 expected_recv=14135 actual_valid_recv=14336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=7552 slack_slots=22144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=14848 expected_recv=19245 expected_overflow=0 exact_recv_per_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=14848 expected_recv=16250 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=14848 expected_recv=18808 expected_overflow=0 exact_recv_per_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=14848 expected_recv=19245 expected_overflow=0 exact_recv_per_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=14336 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=14848 expected_recv=19245 actual_valid_recv=19456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=14848 expected_recv=18808 actual_valid_recv=18944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=14848 expected_recv=19245 actual_valid_recv=19456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=14848 expected_recv=17836 expected_overflow=0 exact_recv_per_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=19456 slack_slots=10240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=18944 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=19456 slack_slots=10240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=14848 expected_recv=13608 expected_overflow=0 exact_recv_per_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=14848 expected_recv=17836 actual_valid_recv=18048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=18048 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=14848 expected_recv=13608 actual_valid_recv=13824 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=14848 expected_recv=15041 expected_overflow=0 exact_recv_per_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=13824 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=14848 expected_recv=15042 expected_overflow=0 exact_recv_per_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=14848 expected_recv=17801 expected_overflow=0 exact_recv_per_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=14848 expected_recv=15041 actual_valid_recv=15232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=14848 expected_recv=15042 actual_valid_recv=15232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=14848 expected_recv=15707 expected_overflow=0 exact_recv_per_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=14848 expected_recv=17801 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=14848 expected_recv=10576 expected_overflow=0 exact_recv_per_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=14848 expected_recv=17229 expected_overflow=0 exact_recv_per_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=15232 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=15232 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=17920 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=14848 expected_recv=15707 actual_valid_recv=16000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=14848 expected_recv=17020 expected_overflow=0 exact_recv_per_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=14848 expected_recv=17229 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=14848 expected_recv=10576 actual_valid_recv=10752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16000 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=17280 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=14848 expected_recv=17020 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=10752 slack_slots=18944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=14848 expected_recv=19509 expected_overflow=0 exact_recv_per_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=17152 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=14848 expected_recv=19509 actual_valid_recv=19712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=14848 expected_recv=21660 expected_overflow=0 exact_recv_per_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=14848 expected_recv=11969 expected_overflow=0 exact_recv_per_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=19712 slack_slots=9984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=14848 expected_recv=21660 actual_valid_recv=21760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=14848 expected_recv=11969 actual_valid_recv=12160 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=14848 expected_recv=9568 expected_overflow=0 exact_recv_per_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=14848 expected_recv=19112 expected_overflow=0 exact_recv_per_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=21760 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=12160 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=14848 expected_recv=16497 expected_overflow=0 exact_recv_per_rank=[9094, 9811, 9421, 6202, 9811, 3549, 9811, 7837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=14848 expected_recv=21658 expected_overflow=0 exact_recv_per_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=14848 expected_recv=9568 actual_valid_recv=9856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=14848 expected_recv=13531 expected_overflow=0 exact_recv_per_rank=[9094, 9811, 9421, 6202, 9811, 3549, 9811, 7837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=14848 expected_recv=19112 actual_valid_recv=19328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=14848 expected_recv=16497 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=9856 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=14848 expected_recv=21658 actual_valid_recv=21760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=14848 expected_recv=13531 actual_valid_recv=13696 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=19328 slack_slots=10368 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=14848 expected_recv=12498 expected_overflow=0 exact_recv_per_rank=[9094, 9811, 9421, 6202, 9811, 3549, 9811, 7837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=21760 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=13696 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=14848 expected_recv=15186 expected_overflow=0 exact_recv_per_rank=[9094, 9811, 9421, 6202, 9811, 3549, 9811, 7837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=14848 expected_recv=12498 actual_valid_recv=12544 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=14848 expected_recv=17694 expected_overflow=0 exact_recv_per_rank=[9094, 9811, 9421, 6202, 9811, 3549, 9811, 7837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=14848 expected_recv=15186 actual_valid_recv=15488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=12544 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=14848 expected_recv=17694 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=15488 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=17792 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=14848 expected_recv=18794 expected_overflow=0 exact_recv_per_rank=[9094, 9811, 9421, 6202, 9811, 3549, 9811, 7837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=14848 expected_recv=14341 expected_overflow=0 exact_recv_per_rank=[8496, 8497, 8496, 7901, 8498, 8207, 6943, 8498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=14848 expected_recv=18078 expected_overflow=0 exact_recv_per_rank=[9094, 9811, 9421, 6202, 9811, 3549, 9811, 7837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=14848 expected_recv=18362 expected_overflow=0 exact_recv_per_rank=[8496, 8497, 8496, 7901, 8498, 8207, 6943, 8498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=14848 expected_recv=18794 expected_overflow=0 exact_recv_per_rank=[9094, 9811, 9421, 6202, 9811, 3549, 9811, 7837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=14848 expected_recv=18794 actual_valid_recv=18944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=14848 expected_recv=14341 actual_valid_recv=14464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=14848 expected_recv=18078 actual_valid_recv=18304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=14848 expected_recv=18069 expected_overflow=0 exact_recv_per_rank=[8496, 8497, 8496, 7901, 8498, 8207, 6943, 8498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=14848 expected_recv=18362 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=18944 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=14848 expected_recv=18794 actual_valid_recv=18944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=14464 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=18304 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=14848 expected_recv=17765 expected_overflow=0 exact_recv_per_rank=[8496, 8497, 8496, 7901, 8498, 8207, 6943, 8498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=18560 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=14848 expected_recv=13837 expected_overflow=0 exact_recv_per_rank=[8496, 8497, 8496, 7901, 8498, 8207, 6943, 8498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=14848 expected_recv=18069 actual_valid_recv=18432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=18944 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=14848 expected_recv=17765 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=18432 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=14848 expected_recv=13837 actual_valid_recv=14080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=17920 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=14080 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=14848 expected_recv=18360 expected_overflow=0 exact_recv_per_rank=[8496, 8497, 8496, 7901, 8498, 8207, 6943, 8498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=14848 expected_recv=14140 expected_overflow=0 exact_recv_per_rank=[9469, 7903, 4799, 9467, 5739, 9468, 9223, 9468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=14848 expected_recv=18361 expected_overflow=0 exact_recv_per_rank=[8496, 8497, 8496, 7901, 8498, 8207, 6943, 8498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=14848 expected_recv=14099 expected_overflow=0 exact_recv_per_rank=[9469, 7903, 4799, 9467, 5739, 9468, 9223, 9468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=14848 expected_recv=11977 expected_overflow=0 exact_recv_per_rank=[8496, 8497, 8496, 7901, 8498, 8207, 6943, 8498] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=14848 expected_recv=18360 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=14848 expected_recv=18361 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=14848 expected_recv=14140 actual_valid_recv=14336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=14848 expected_recv=14099 actual_valid_recv=14464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=14848 expected_recv=18123 expected_overflow=0 exact_recv_per_rank=[9469, 7903, 4799, 9467, 5739, 9468, 9223, 9468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=18560 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=18560 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=14848 expected_recv=11977 actual_valid_recv=12160 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=14336 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=14848 expected_recv=19231 expected_overflow=0 exact_recv_per_rank=[9469, 7903, 4799, 9467, 5739, 9468, 9223, 9468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=14464 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=14848 expected_recv=18123 actual_valid_recv=18176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=12160 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=18176 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=14848 expected_recv=19231 actual_valid_recv=19328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=14848 expected_recv=15503 expected_overflow=0 exact_recv_per_rank=[9469, 7903, 4799, 9467, 5739, 9468, 9223, 9468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=19328 slack_slots=10368 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=14848 expected_recv=15503 actual_valid_recv=15616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=14848 expected_recv=11755 expected_overflow=0 exact_recv_per_rank=[9469, 7903, 4799, 9467, 5739, 9468, 9223, 9468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=15616 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=14848 expected_recv=19234 expected_overflow=0 exact_recv_per_rank=[9469, 7903, 4799, 9467, 5739, 9468, 9223, 9468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=14848 expected_recv=13379 expected_overflow=0 exact_recv_per_rank=[9360, 10475, 4842, 10475, 5183, 4371, 10355, 10475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=14848 expected_recv=18438 expected_overflow=0 exact_recv_per_rank=[9360, 10475, 4842, 10475, 5183, 4371, 10355, 10475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=14848 expected_recv=11755 actual_valid_recv=12032 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=14848 expected_recv=18987 expected_overflow=0 exact_recv_per_rank=[9469, 7903, 4799, 9467, 5739, 9468, 9223, 9468] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=14848 expected_recv=19234 actual_valid_recv=19328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=14848 expected_recv=13379 actual_valid_recv=13568 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=14848 expected_recv=18438 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=14848 expected_recv=12964 expected_overflow=0 exact_recv_per_rank=[9360, 10475, 4842, 10475, 5183, 4371, 10355, 10475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=12032 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=19328 slack_slots=10368 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=14848 expected_recv=18987 actual_valid_recv=19200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=14848 expected_recv=19068 expected_overflow=0 exact_recv_per_rank=[9360, 10475, 4842, 10475, 5183, 4371, 10355, 10475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=13568 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=18560 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=14848 expected_recv=12964 actual_valid_recv=13184 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=19200 slack_slots=10496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=14848 expected_recv=19068 actual_valid_recv=19200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=14848 expected_recv=13420 expected_overflow=0 exact_recv_per_rank=[9360, 10475, 4842, 10475, 5183, 4371, 10355, 10475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=13184 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=19200 slack_slots=10496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=14848 expected_recv=13420 actual_valid_recv=13568 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=14848 expected_recv=19070 expected_overflow=0 exact_recv_per_rank=[9360, 10475, 4842, 10475, 5183, 4371, 10355, 10475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=14848 expected_recv=17954 expected_overflow=0 exact_recv_per_rank=[9360, 10475, 4842, 10475, 5183, 4371, 10355, 10475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=14848 expected_recv=20446 expected_overflow=0 exact_recv_per_rank=[9460, 5720, 9460, 9458, 9459, 5180, 7340, 9459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=13568 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=14848 expected_recv=19070 actual_valid_recv=19200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=14848 expected_recv=17954 actual_valid_recv=18176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=14848 expected_recv=20207 expected_overflow=0 exact_recv_per_rank=[9460, 5720, 9460, 9458, 9459, 5180, 7340, 9459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=14848 expected_recv=16779 expected_overflow=0 exact_recv_per_rank=[9360, 10475, 4842, 10475, 5183, 4371, 10355, 10475] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=14848 expected_recv=20446 actual_valid_recv=20480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=18176 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=19200 slack_slots=10496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=14848 expected_recv=20207 actual_valid_recv=20224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=14848 expected_recv=16779 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=20480 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=14848 expected_recv=12203 expected_overflow=0 exact_recv_per_rank=[9460, 5720, 9460, 9458, 9459, 5180, 7340, 9459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=20224 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=14848 expected_recv=22014 expected_overflow=0 exact_recv_per_rank=[9460, 5720, 9460, 9458, 9459, 5180, 7340, 9459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=17152 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=14848 expected_recv=12203 actual_valid_recv=12544 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=14848 expected_recv=22014 actual_valid_recv=22144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=12544 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=14848 expected_recv=22015 expected_overflow=0 exact_recv_per_rank=[9460, 5720, 9460, 9458, 9459, 5180, 7340, 9459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=22144 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=14848 expected_recv=22015 actual_valid_recv=22144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=14848 expected_recv=15213 expected_overflow=0 exact_recv_per_rank=[9460, 5720, 9460, 9458, 9459, 5180, 7340, 9459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=14848 expected_recv=9357 expected_overflow=0 exact_recv_per_rank=[9460, 5720, 9460, 9458, 9459, 5180, 7340, 9459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=14848 expected_recv=17840 expected_overflow=0 exact_recv_per_rank=[9253, 7241, 7334, 9254, 6000, 9255, 7944, 9255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=22144 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=14848 expected_recv=15829 expected_overflow=0 exact_recv_per_rank=[9253, 7241, 7334, 9254, 6000, 9255, 7944, 9255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=14848 expected_recv=15213 actual_valid_recv=15360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=14848 expected_recv=9357 actual_valid_recv=9600 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=14848 expected_recv=17840 actual_valid_recv=18048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=14848 expected_recv=9617 expected_overflow=0 exact_recv_per_rank=[9460, 5720, 9460, 9458, 9459, 5180, 7340, 9459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=14848 expected_recv=15829 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=15360 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=14848 expected_recv=17839 expected_overflow=0 exact_recv_per_rank=[9253, 7241, 7334, 9254, 6000, 9255, 7944, 9255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=18048 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=9600 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=14848 expected_recv=17838 expected_overflow=0 exact_recv_per_rank=[9253, 7241, 7334, 9254, 6000, 9255, 7944, 9255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=14848 expected_recv=9617 actual_valid_recv=9856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=14848 expected_recv=17839 actual_valid_recv=18048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=9856 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=14848 expected_recv=17838 actual_valid_recv=18048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=18048 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=14848 expected_recv=14583 expected_overflow=0 exact_recv_per_rank=[9253, 7241, 7334, 9254, 6000, 9255, 7944, 9255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=18048 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=14848 expected_recv=14583 actual_valid_recv=14848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=14848 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=14848 expected_recv=14827 expected_overflow=0 exact_recv_per_rank=[9253, 7241, 7334, 9254, 6000, 9255, 7944, 9255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=14848 expected_recv=17310 expected_overflow=0 exact_recv_per_rank=[5752, 10810, 6294, 7767, 5460, 10811, 7832, 10810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=14848 expected_recv=11353 expected_overflow=0 exact_recv_per_rank=[5752, 10810, 6294, 7767, 5460, 10811, 7832, 10810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=14848 expected_recv=15787 expected_overflow=0 exact_recv_per_rank=[9253, 7241, 7334, 9254, 6000, 9255, 7944, 9255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=14848 expected_recv=14827 actual_valid_recv=15104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=14848 expected_recv=16529 expected_overflow=0 exact_recv_per_rank=[9253, 7241, 7334, 9254, 6000, 9255, 7944, 9255] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=14848 expected_recv=17310 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=14848 expected_recv=21683 expected_overflow=0 exact_recv_per_rank=[5752, 10810, 6294, 7767, 5460, 10811, 7832, 10810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=14848 expected_recv=11353 actual_valid_recv=11520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=14848 expected_recv=15787 actual_valid_recv=16000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=15104 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=14848 expected_recv=16529 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=14848 expected_recv=18639 expected_overflow=0 exact_recv_per_rank=[5752, 10810, 6294, 7767, 5460, 10811, 7832, 10810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=17408 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=14848 expected_recv=21683 actual_valid_recv=21760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=11520 slack_slots=18176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16000 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=14848 expected_recv=13861 expected_overflow=0 exact_recv_per_rank=[5752, 10810, 6294, 7767, 5460, 10811, 7832, 10810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=14848 expected_recv=18639 actual_valid_recv=18944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=21760 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=14848 expected_recv=13861 actual_valid_recv=13952 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=18944 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=13952 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=14848 expected_recv=9425 expected_overflow=0 exact_recv_per_rank=[5752, 10810, 6294, 7767, 5460, 10811, 7832, 10810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=14848 expected_recv=21682 expected_overflow=0 exact_recv_per_rank=[5752, 10810, 6294, 7767, 5460, 10811, 7832, 10810] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[880, 26566, 4616, 2953, 11158, 3239, 6042, 10082] avg=8192.0000 max=26566 max_rank=1 min=880 min_rank=0 imbalance_ratio=3.242920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=14848 expected_recv=17119 expected_overflow=0 exact_recv_per_rank=[5752, 10810, 6294, 7767, 5460, 10811, 7832, 10810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=14848 expected_recv=9425 actual_valid_recv=9600 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[880, 26566, 4616, 2953, 11158, 3239, 6042, 10082] avg=8192.0000 max=26566 max_rank=1 min=880 min_rank=0 imbalance_ratio=3.242920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=14848 expected_recv=21682 actual_valid_recv=21760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=9600 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=14848 expected_recv=17119 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[880, 26566, 4616, 2953, 11158, 3239, 6042, 10082] avg=8192.0000 max=26566 max_rank=1 min=880 min_rank=0 imbalance_ratio=3.242920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=21760 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=17280 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[880, 26566, 4616, 2953, 11158, 3239, 6042, 10082] avg=8192.0000 max=26566 max_rank=1 min=880 min_rank=0 imbalance_ratio=3.242920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] avg=8192.0000 max=8541 max_rank=0 min=7234 min_rank=2 imbalance_ratio=1.042603 max_load_reduction=18025 max_load_reduction_pct=0.678499 moved_to_replicas=43107 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] avg=8192.0000 max=8541 max_rank=0 min=7234 min_rank=2 imbalance_ratio=1.042603 max_load_reduction=18025 max_load_reduction_pct=0.678499 moved_to_replicas=43107 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[880, 26566, 4616, 2953, 11158, 3239, 6042, 10082] avg=8192.0000 max=26566 max_rank=1 min=880 min_rank=0 imbalance_ratio=3.242920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] avg=8192.0000 max=8541 max_rank=0 min=7234 min_rank=2 imbalance_ratio=1.042603 max_load_reduction=18025 max_load_reduction_pct=0.678499 moved_to_replicas=43107 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] avg=8192.0000 max=8541 max_rank=0 min=7234 min_rank=2 imbalance_ratio=1.042603 max_load_reduction=18025 max_load_reduction_pct=0.678499 moved_to_replicas=43107 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[880, 26566, 4616, 2953, 11158, 3239, 6042, 10082] avg=8192.0000 max=26566 max_rank=1 min=880 min_rank=0 imbalance_ratio=3.242920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[880, 26566, 4616, 2953, 11158, 3239, 6042, 10082] avg=8192.0000 max=26566 max_rank=1 min=880 min_rank=0 imbalance_ratio=3.242920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] avg=8192.0000 max=8541 max_rank=0 min=7234 min_rank=2 imbalance_ratio=1.042603 max_load_reduction=18025 max_load_reduction_pct=0.678499 moved_to_replicas=43107 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[880, 26566, 4616, 2953, 11158, 3239, 6042, 10082] avg=8192.0000 max=26566 max_rank=1 min=880 min_rank=0 imbalance_ratio=3.242920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[8102, 1081, 2303, 7902, 17066, 4358, 11235, 13489] avg=8192.0000 max=17066 max_rank=4 min=1081 min_rank=1 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[8102, 1081, 2303, 7902, 17066, 4358, 11235, 13489] avg=8192.0000 max=17066 max_rank=4 min=1081 min_rank=1 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[8102, 1081, 2303, 7902, 17066, 4358, 11235, 13489] avg=8192.0000 max=17066 max_rank=4 min=1081 min_rank=1 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] avg=8192.0000 max=8541 max_rank=0 min=7234 min_rank=2 imbalance_ratio=1.042603 max_load_reduction=18025 max_load_reduction_pct=0.678499 moved_to_replicas=43107 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] avg=8192.0000 max=8541 max_rank=0 min=7234 min_rank=2 imbalance_ratio=1.042603 max_load_reduction=18025 max_load_reduction_pct=0.678499 moved_to_replicas=43107 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[8102, 1081, 2303, 7902, 17066, 4358, 11235, 13489] avg=8192.0000 max=17066 max_rank=4 min=1081 min_rank=1 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] avg=8192.0000 max=9075 max_rank=6 min=6518 min_rank=5 imbalance_ratio=1.107788 max_load_reduction=7991 max_load_reduction_pct=0.468241 moved_to_replicas=19465 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] avg=8192.0000 max=9075 max_rank=6 min=6518 min_rank=5 imbalance_ratio=1.107788 max_load_reduction=7991 max_load_reduction_pct=0.468241 moved_to_replicas=19465 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[8541, 8539, 7234, 8540, 8540, 8018, 7584, 8540] avg=8192.0000 max=8541 max_rank=0 min=7234 min_rank=2 imbalance_ratio=1.042603 max_load_reduction=18025 max_load_reduction_pct=0.678499 moved_to_replicas=43107 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] avg=8192.0000 max=9075 max_rank=6 min=6518 min_rank=5 imbalance_ratio=1.107788 max_load_reduction=7991 max_load_reduction_pct=0.468241 moved_to_replicas=19465 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[8102, 1081, 2303, 7902, 17066, 4358, 11235, 13489] avg=8192.0000 max=17066 max_rank=4 min=1081 min_rank=1 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] avg=8192.0000 max=9075 max_rank=6 min=6518 min_rank=5 imbalance_ratio=1.107788 max_load_reduction=7991 max_load_reduction_pct=0.468241 moved_to_replicas=19465 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[8102, 1081, 2303, 7902, 17066, 4358, 11235, 13489] avg=8192.0000 max=17066 max_rank=4 min=1081 min_rank=1 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] avg=8192.0000 max=9075 max_rank=6 min=6518 min_rank=5 imbalance_ratio=1.107788 max_load_reduction=7991 max_load_reduction_pct=0.468241 moved_to_replicas=19465 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[8102, 1081, 2303, 7902, 17066, 4358, 11235, 13489] avg=8192.0000 max=17066 max_rank=4 min=1081 min_rank=1 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[11173, 1843, 8958, 17066, 1359, 12417, 8436, 4284] avg=8192.0000 max=17066 max_rank=3 min=1359 min_rank=4 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[11173, 1843, 8958, 17066, 1359, 12417, 8436, 4284] avg=8192.0000 max=17066 max_rank=3 min=1359 min_rank=4 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[8102, 1081, 2303, 7902, 17066, 4358, 11235, 13489] avg=8192.0000 max=17066 max_rank=4 min=1081 min_rank=1 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[11173, 1843, 8958, 17066, 1359, 12417, 8436, 4284] avg=8192.0000 max=17066 max_rank=3 min=1359 min_rank=4 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] avg=8192.0000 max=9075 max_rank=6 min=6518 min_rank=5 imbalance_ratio=1.107788 max_load_reduction=7991 max_load_reduction_pct=0.468241 moved_to_replicas=19465 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[11173, 1843, 8958, 17066, 1359, 12417, 8436, 4284] avg=8192.0000 max=17066 max_rank=3 min=1359 min_rank=4 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] avg=8192.0000 max=9213 max_rank=0 min=5048 min_rank=1 imbalance_ratio=1.124634 max_load_reduction=7853 max_load_reduction_pct=0.460155 moved_to_replicas=24076 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] avg=8192.0000 max=9075 max_rank=6 min=6518 min_rank=5 imbalance_ratio=1.107788 max_load_reduction=7991 max_load_reduction_pct=0.468241 moved_to_replicas=19465 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] avg=8192.0000 max=9213 max_rank=0 min=5048 min_rank=1 imbalance_ratio=1.124634 max_load_reduction=7853 max_load_reduction_pct=0.460155 moved_to_replicas=24076 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[8102, 9073, 6718, 7902, 9074, 6518, 9075, 9074] avg=8192.0000 max=9075 max_rank=6 min=6518 min_rank=5 imbalance_ratio=1.107788 max_load_reduction=7991 max_load_reduction_pct=0.468241 moved_to_replicas=19465 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[11173, 1843, 8958, 17066, 1359, 12417, 8436, 4284] avg=8192.0000 max=17066 max_rank=3 min=1359 min_rank=4 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] avg=8192.0000 max=9213 max_rank=0 min=5048 min_rank=1 imbalance_ratio=1.124634 max_load_reduction=7853 max_load_reduction_pct=0.460155 moved_to_replicas=24076 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] avg=8192.0000 max=9213 max_rank=0 min=5048 min_rank=1 imbalance_ratio=1.124634 max_load_reduction=7853 max_load_reduction_pct=0.460155 moved_to_replicas=24076 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] avg=8192.0000 max=9213 max_rank=0 min=5048 min_rank=1 imbalance_ratio=1.124634 max_load_reduction=7853 max_load_reduction_pct=0.460155 moved_to_replicas=24076 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[9087, 21876, 1721, 7065, 7819, 3960, 7691, 6317] avg=8192.0000 max=21876 max_rank=1 min=1721 min_rank=2 imbalance_ratio=2.670410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[9087, 21876, 1721, 7065, 7819, 3960, 7691, 6317] avg=8192.0000 max=21876 max_rank=1 min=1721 min_rank=2 imbalance_ratio=2.670410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[11173, 1843, 8958, 17066, 1359, 12417, 8436, 4284] avg=8192.0000 max=17066 max_rank=3 min=1359 min_rank=4 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[11173, 1843, 8958, 17066, 1359, 12417, 8436, 4284] avg=8192.0000 max=17066 max_rank=3 min=1359 min_rank=4 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[11173, 1843, 8958, 17066, 1359, 12417, 8436, 4284] avg=8192.0000 max=17066 max_rank=3 min=1359 min_rank=4 imbalance_ratio=2.083252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[9087, 21876, 1721, 7065, 7819, 3960, 7691, 6317] avg=8192.0000 max=21876 max_rank=1 min=1721 min_rank=2 imbalance_ratio=2.670410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[9087, 21876, 1721, 7065, 7819, 3960, 7691, 6317] avg=8192.0000 max=21876 max_rank=1 min=1721 min_rank=2 imbalance_ratio=2.670410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] avg=8192.0000 max=8274 max_rank=0 min=7843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13602 max_load_reduction_pct=0.621777 moved_to_replicas=33488 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] avg=8192.0000 max=8274 max_rank=0 min=7843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13602 max_load_reduction_pct=0.621777 moved_to_replicas=33488 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] avg=8192.0000 max=9213 max_rank=0 min=5048 min_rank=1 imbalance_ratio=1.124634 max_load_reduction=7853 max_load_reduction_pct=0.460155 moved_to_replicas=24076 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] avg=8192.0000 max=9213 max_rank=0 min=5048 min_rank=1 imbalance_ratio=1.124634 max_load_reduction=7853 max_load_reduction_pct=0.460155 moved_to_replicas=24076 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[9213, 5048, 8958, 9212, 9213, 9212, 8436, 6244] avg=8192.0000 max=9213 max_rank=0 min=5048 min_rank=1 imbalance_ratio=1.124634 max_load_reduction=7853 max_load_reduction_pct=0.460155 moved_to_replicas=24076 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[9087, 21876, 1721, 7065, 7819, 3960, 7691, 6317] avg=8192.0000 max=21876 max_rank=1 min=1721 min_rank=2 imbalance_ratio=2.670410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] avg=8192.0000 max=8274 max_rank=0 min=7843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13602 max_load_reduction_pct=0.621777 moved_to_replicas=33488 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] avg=8192.0000 max=8274 max_rank=0 min=7843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13602 max_load_reduction_pct=0.621777 moved_to_replicas=33488 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[15475, 6841, 6249, 8271, 6238, 6916, 6203, 9343] avg=8192.0000 max=15475 max_rank=0 min=6203 min_rank=6 imbalance_ratio=1.889038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[15475, 6841, 6249, 8271, 6238, 6916, 6203, 9343] avg=8192.0000 max=15475 max_rank=0 min=6203 min_rank=6 imbalance_ratio=1.889038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] avg=8192.0000 max=8274 max_rank=0 min=7843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13602 max_load_reduction_pct=0.621777 moved_to_replicas=33488 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[9087, 21876, 1721, 7065, 7819, 3960, 7691, 6317] avg=8192.0000 max=21876 max_rank=1 min=1721 min_rank=2 imbalance_ratio=2.670410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[9087, 21876, 1721, 7065, 7819, 3960, 7691, 6317] avg=8192.0000 max=21876 max_rank=1 min=1721 min_rank=2 imbalance_ratio=2.670410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[9087, 21876, 1721, 7065, 7819, 3960, 7691, 6317] avg=8192.0000 max=21876 max_rank=1 min=1721 min_rank=2 imbalance_ratio=2.670410 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[15475, 6841, 6249, 8271, 6238, 6916, 6203, 9343] avg=8192.0000 max=15475 max_rank=0 min=6203 min_rank=6 imbalance_ratio=1.889038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[15475, 6841, 6249, 8271, 6238, 6916, 6203, 9343] avg=8192.0000 max=15475 max_rank=0 min=6203 min_rank=6 imbalance_ratio=1.889038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] avg=8192.0000 max=8274 max_rank=2 min=7911 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7201 max_load_reduction_pct=0.465331 moved_to_replicas=13873 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] avg=8192.0000 max=8274 max_rank=2 min=7911 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7201 max_load_reduction_pct=0.465331 moved_to_replicas=13873 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] avg=8192.0000 max=8274 max_rank=0 min=7843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13602 max_load_reduction_pct=0.621777 moved_to_replicas=33488 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] avg=8192.0000 max=8274 max_rank=0 min=7843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13602 max_load_reduction_pct=0.621777 moved_to_replicas=33488 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[8274, 8274, 8274, 7843, 8049, 8274, 8274, 8274] avg=8192.0000 max=8274 max_rank=0 min=7843 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13602 max_load_reduction_pct=0.621777 moved_to_replicas=33488 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[15475, 6841, 6249, 8271, 6238, 6916, 6203, 9343] avg=8192.0000 max=15475 max_rank=0 min=6203 min_rank=6 imbalance_ratio=1.889038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] avg=8192.0000 max=8274 max_rank=2 min=7911 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7201 max_load_reduction_pct=0.465331 moved_to_replicas=13873 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] avg=8192.0000 max=8274 max_rank=2 min=7911 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7201 max_load_reduction_pct=0.465331 moved_to_replicas=13873 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[5092, 6103, 5082, 7798, 17478, 9628, 6699, 7656] avg=8192.0000 max=17478 max_rank=4 min=5082 min_rank=2 imbalance_ratio=2.133545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] avg=8192.0000 max=8274 max_rank=2 min=7911 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7201 max_load_reduction_pct=0.465331 moved_to_replicas=13873 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[15475, 6841, 6249, 8271, 6238, 6916, 6203, 9343] avg=8192.0000 max=15475 max_rank=0 min=6203 min_rank=6 imbalance_ratio=1.889038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[5092, 6103, 5082, 7798, 17478, 9628, 6699, 7656] avg=8192.0000 max=17478 max_rank=4 min=5082 min_rank=2 imbalance_ratio=2.133545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[15475, 6841, 6249, 8271, 6238, 6916, 6203, 9343] avg=8192.0000 max=15475 max_rank=0 min=6203 min_rank=6 imbalance_ratio=1.889038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[15475, 6841, 6249, 8271, 6238, 6916, 6203, 9343] avg=8192.0000 max=15475 max_rank=0 min=6203 min_rank=6 imbalance_ratio=1.889038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[5092, 6103, 5082, 7798, 17478, 9628, 6699, 7656] avg=8192.0000 max=17478 max_rank=4 min=5082 min_rank=2 imbalance_ratio=2.133545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[5092, 6103, 5082, 7798, 17478, 9628, 6699, 7656] avg=8192.0000 max=17478 max_rank=4 min=5082 min_rank=2 imbalance_ratio=2.133545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] avg=8192.0000 max=8450 max_rank=2 min=7656 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=9028 max_load_reduction_pct=0.516535 moved_to_replicas=22131 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] avg=8192.0000 max=8274 max_rank=2 min=7911 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7201 max_load_reduction_pct=0.465331 moved_to_replicas=13873 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] avg=8192.0000 max=8450 max_rank=2 min=7656 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=9028 max_load_reduction_pct=0.516535 moved_to_replicas=22131 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] avg=8192.0000 max=8274 max_rank=2 min=7911 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7201 max_load_reduction_pct=0.465331 moved_to_replicas=13873 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[8273, 7911, 8274, 8271, 8274, 7985, 8274, 8274] avg=8192.0000 max=8274 max_rank=2 min=7911 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7201 max_load_reduction_pct=0.465331 moved_to_replicas=13873 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[5092, 6103, 5082, 7798, 17478, 9628, 6699, 7656] avg=8192.0000 max=17478 max_rank=4 min=5082 min_rank=2 imbalance_ratio=2.133545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] avg=8192.0000 max=8450 max_rank=2 min=7656 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=9028 max_load_reduction_pct=0.516535 moved_to_replicas=22131 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] avg=8192.0000 max=8450 max_rank=2 min=7656 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=9028 max_load_reduction_pct=0.516535 moved_to_replicas=22131 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[17476, 826, 1489, 1964, 1703, 7237, 16410, 18431] avg=8192.0000 max=18431 max_rank=7 min=826 min_rank=1 imbalance_ratio=2.249878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] avg=8192.0000 max=8450 max_rank=2 min=7656 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=9028 max_load_reduction_pct=0.516535 moved_to_replicas=22131 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[17476, 826, 1489, 1964, 1703, 7237, 16410, 18431] avg=8192.0000 max=18431 max_rank=7 min=826 min_rank=1 imbalance_ratio=2.249878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[5092, 6103, 5082, 7798, 17478, 9628, 6699, 7656] avg=8192.0000 max=17478 max_rank=4 min=5082 min_rank=2 imbalance_ratio=2.133545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[5092, 6103, 5082, 7798, 17478, 9628, 6699, 7656] avg=8192.0000 max=17478 max_rank=4 min=5082 min_rank=2 imbalance_ratio=2.133545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[5092, 6103, 5082, 7798, 17478, 9628, 6699, 7656] avg=8192.0000 max=17478 max_rank=4 min=5082 min_rank=2 imbalance_ratio=2.133545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] avg=8192.0000 max=9630 max_rank=0 min=1964 min_rank=3 imbalance_ratio=1.175537 max_load_reduction=8801 max_load_reduction_pct=0.477511 moved_to_replicas=38197 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[17476, 826, 1489, 1964, 1703, 7237, 16410, 18431] avg=8192.0000 max=18431 max_rank=7 min=826 min_rank=1 imbalance_ratio=2.249878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[17476, 826, 1489, 1964, 1703, 7237, 16410, 18431] avg=8192.0000 max=18431 max_rank=7 min=826 min_rank=1 imbalance_ratio=2.249878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] avg=8192.0000 max=9630 max_rank=0 min=1964 min_rank=3 imbalance_ratio=1.175537 max_load_reduction=8801 max_load_reduction_pct=0.477511 moved_to_replicas=38197 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] avg=8192.0000 max=8450 max_rank=2 min=7656 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=9028 max_load_reduction_pct=0.516535 moved_to_replicas=22131 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[17476, 826, 1489, 1964, 1703, 7237, 16410, 18431] avg=8192.0000 max=18431 max_rank=7 min=826 min_rank=1 imbalance_ratio=2.249878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] avg=8192.0000 max=8450 max_rank=2 min=7656 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=9028 max_load_reduction_pct=0.516535 moved_to_replicas=22131 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[8449, 8408, 8450, 7798, 8448, 8448, 7879, 7656] avg=8192.0000 max=8450 max_rank=2 min=7656 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=9028 max_load_reduction_pct=0.516535 moved_to_replicas=22131 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] avg=8192.0000 max=9630 max_rank=0 min=1964 min_rank=3 imbalance_ratio=1.175537 max_load_reduction=8801 max_load_reduction_pct=0.477511 moved_to_replicas=38197 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] avg=8192.0000 max=9630 max_rank=0 min=1964 min_rank=3 imbalance_ratio=1.175537 max_load_reduction=8801 max_load_reduction_pct=0.477511 moved_to_replicas=38197 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[14663, 5495, 14370, 4817, 10121, 6131, 2345, 7594] avg=8192.0000 max=14663 max_rank=0 min=2345 min_rank=6 imbalance_ratio=1.789917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] avg=8192.0000 max=9630 max_rank=0 min=1964 min_rank=3 imbalance_ratio=1.175537 max_load_reduction=8801 max_load_reduction_pct=0.477511 moved_to_replicas=38197 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[14663, 5495, 14370, 4817, 10121, 6131, 2345, 7594] avg=8192.0000 max=14663 max_rank=0 min=2345 min_rank=6 imbalance_ratio=1.789917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[17476, 826, 1489, 1964, 1703, 7237, 16410, 18431] avg=8192.0000 max=18431 max_rank=7 min=826 min_rank=1 imbalance_ratio=2.249878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[17476, 826, 1489, 1964, 1703, 7237, 16410, 18431] avg=8192.0000 max=18431 max_rank=7 min=826 min_rank=1 imbalance_ratio=2.249878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[14663, 5495, 14370, 4817, 10121, 6131, 2345, 7594] avg=8192.0000 max=14663 max_rank=0 min=2345 min_rank=6 imbalance_ratio=1.789917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[14663, 5495, 14370, 4817, 10121, 6131, 2345, 7594] avg=8192.0000 max=14663 max_rank=0 min=2345 min_rank=6 imbalance_ratio=1.789917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[17476, 826, 1489, 1964, 1703, 7237, 16410, 18431] avg=8192.0000 max=18431 max_rank=7 min=826 min_rank=1 imbalance_ratio=2.249878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] avg=8192.0000 max=8569 max_rank=3 min=7546 min_rank=1 imbalance_ratio=1.046021 max_load_reduction=6094 max_load_reduction_pct=0.415604 moved_to_replicas=19760 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] avg=8192.0000 max=8569 max_rank=3 min=7546 min_rank=1 imbalance_ratio=1.046021 max_load_reduction=6094 max_load_reduction_pct=0.415604 moved_to_replicas=19760 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] avg=8192.0000 max=9630 max_rank=0 min=1964 min_rank=3 imbalance_ratio=1.175537 max_load_reduction=8801 max_load_reduction_pct=0.477511 moved_to_replicas=38197 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[14663, 5495, 14370, 4817, 10121, 6131, 2345, 7594] avg=8192.0000 max=14663 max_rank=0 min=2345 min_rank=6 imbalance_ratio=1.789917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] avg=8192.0000 max=9630 max_rank=0 min=1964 min_rank=3 imbalance_ratio=1.175537 max_load_reduction=8801 max_load_reduction_pct=0.477511 moved_to_replicas=38197 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] avg=8192.0000 max=8569 max_rank=3 min=7546 min_rank=1 imbalance_ratio=1.046021 max_load_reduction=6094 max_load_reduction_pct=0.415604 moved_to_replicas=19760 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] avg=8192.0000 max=8569 max_rank=3 min=7546 min_rank=1 imbalance_ratio=1.046021 max_load_reduction=6094 max_load_reduction_pct=0.415604 moved_to_replicas=19760 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[9630, 9628, 9335, 1964, 8484, 7237, 9629, 9629] avg=8192.0000 max=9630 max_rank=0 min=1964 min_rank=3 imbalance_ratio=1.175537 max_load_reduction=8801 max_load_reduction_pct=0.477511 moved_to_replicas=38197 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] avg=8192.0000 max=8569 max_rank=3 min=7546 min_rank=1 imbalance_ratio=1.046021 max_load_reduction=6094 max_load_reduction_pct=0.415604 moved_to_replicas=19760 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 20789, 2987, 1890, 12413, 10818] avg=8192.0000 max=20789 max_rank=3 min=1890 min_rank=5 imbalance_ratio=2.537720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 20789, 2987, 1890, 12413, 10818] avg=8192.0000 max=20789 max_rank=3 min=1890 min_rank=5 imbalance_ratio=2.537720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[14663, 5495, 14370, 4817, 10121, 6131, 2345, 7594] avg=8192.0000 max=14663 max_rank=0 min=2345 min_rank=6 imbalance_ratio=1.789917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[14663, 5495, 14370, 4817, 10121, 6131, 2345, 7594] avg=8192.0000 max=14663 max_rank=0 min=2345 min_rank=6 imbalance_ratio=1.789917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 20789, 2987, 1890, 12413, 10818] avg=8192.0000 max=20789 max_rank=3 min=1890 min_rank=5 imbalance_ratio=2.537720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 20789, 2987, 1890, 12413, 10818] avg=8192.0000 max=20789 max_rank=3 min=1890 min_rank=5 imbalance_ratio=2.537720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[14663, 5495, 14370, 4817, 10121, 6131, 2345, 7594] avg=8192.0000 max=14663 max_rank=0 min=2345 min_rank=6 imbalance_ratio=1.789917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] avg=8192.0000 max=12413 max_rank=3 min=2820 min_rank=1 imbalance_ratio=1.515259 max_load_reduction=8376 max_load_reduction_pct=0.402905 moved_to_replicas=20831 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] avg=8192.0000 max=12413 max_rank=3 min=2820 min_rank=1 imbalance_ratio=1.515259 max_load_reduction=8376 max_load_reduction_pct=0.402905 moved_to_replicas=20831 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 20789, 2987, 1890, 12413, 10818] avg=8192.0000 max=20789 max_rank=3 min=1890 min_rank=5 imbalance_ratio=2.537720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] avg=8192.0000 max=8569 max_rank=3 min=7546 min_rank=1 imbalance_ratio=1.046021 max_load_reduction=6094 max_load_reduction_pct=0.415604 moved_to_replicas=19760 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] avg=8192.0000 max=12413 max_rank=3 min=2820 min_rank=1 imbalance_ratio=1.515259 max_load_reduction=8376 max_load_reduction_pct=0.402905 moved_to_replicas=20831 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] avg=8192.0000 max=8569 max_rank=3 min=7546 min_rank=1 imbalance_ratio=1.046021 max_load_reduction=6094 max_load_reduction_pct=0.415604 moved_to_replicas=19760 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] avg=8192.0000 max=12413 max_rank=3 min=2820 min_rank=1 imbalance_ratio=1.515259 max_load_reduction=8376 max_load_reduction_pct=0.402905 moved_to_replicas=20831 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[8568, 7546, 8567, 8569, 8569, 7683, 8440, 7594] avg=8192.0000 max=8569 max_rank=3 min=7546 min_rank=1 imbalance_ratio=1.046021 max_load_reduction=6094 max_load_reduction_pct=0.415604 moved_to_replicas=19760 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[9094, 17628, 1604, 6202, 2647, 3549, 16975, 7837] avg=8192.0000 max=17628 max_rank=1 min=1604 min_rank=2 imbalance_ratio=2.151855 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] avg=8192.0000 max=12413 max_rank=3 min=2820 min_rank=1 imbalance_ratio=1.515259 max_load_reduction=8376 max_load_reduction_pct=0.402905 moved_to_replicas=20831 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[9094, 17628, 1604, 6202, 2647, 3549, 16975, 7837] avg=8192.0000 max=17628 max_rank=1 min=1604 min_rank=2 imbalance_ratio=2.151855 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 20789, 2987, 1890, 12413, 10818] avg=8192.0000 max=20789 max_rank=3 min=1890 min_rank=5 imbalance_ratio=2.537720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[9094, 17628, 1604, 6202, 2647, 3549, 16975, 7837] avg=8192.0000 max=17628 max_rank=1 min=1604 min_rank=2 imbalance_ratio=2.151855 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 20789, 2987, 1890, 12413, 10818] avg=8192.0000 max=20789 max_rank=3 min=1890 min_rank=5 imbalance_ratio=2.537720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[9094, 9811, 9421, 6202, 9811, 3549, 9811, 7837] avg=8192.0000 max=9811 max_rank=1 min=3549 min_rank=5 imbalance_ratio=1.197632 max_load_reduction=7817 max_load_reduction_pct=0.443442 moved_to_replicas=29257 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[9094, 17628, 1604, 6202, 2647, 3549, 16975, 7837] avg=8192.0000 max=17628 max_rank=1 min=1604 min_rank=2 imbalance_ratio=2.151855 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 20789, 2987, 1890, 12413, 10818] avg=8192.0000 max=20789 max_rank=3 min=1890 min_rank=5 imbalance_ratio=2.537720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[9094, 9811, 9421, 6202, 9811, 3549, 9811, 7837] avg=8192.0000 max=9811 max_rank=1 min=3549 min_rank=5 imbalance_ratio=1.197632 max_load_reduction=7817 max_load_reduction_pct=0.443442 moved_to_replicas=29257 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[9094, 17628, 1604, 6202, 2647, 3549, 16975, 7837] avg=8192.0000 max=17628 max_rank=1 min=1604 min_rank=2 imbalance_ratio=2.151855 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] avg=8192.0000 max=12413 max_rank=3 min=2820 min_rank=1 imbalance_ratio=1.515259 max_load_reduction=8376 max_load_reduction_pct=0.402905 moved_to_replicas=20831 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[9094, 9811, 9421, 6202, 9811, 3549, 9811, 7837] avg=8192.0000 max=9811 max_rank=1 min=3549 min_rank=5 imbalance_ratio=1.197632 max_load_reduction=7817 max_load_reduction_pct=0.443442 moved_to_replicas=29257 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] avg=8192.0000 max=12413 max_rank=3 min=2820 min_rank=1 imbalance_ratio=1.515259 max_load_reduction=8376 max_load_reduction_pct=0.402905 moved_to_replicas=20831 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[9094, 9811, 9421, 6202, 9811, 3549, 9811, 7837] avg=8192.0000 max=9811 max_rank=1 min=3549 min_rank=5 imbalance_ratio=1.197632 max_load_reduction=7817 max_load_reduction_pct=0.443442 moved_to_replicas=29257 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[9868, 2820, 3951, 12413, 2987, 10266, 12413, 10818] avg=8192.0000 max=12413 max_rank=3 min=2820 min_rank=1 imbalance_ratio=1.515259 max_load_reduction=8376 max_load_reduction_pct=0.402905 moved_to_replicas=20831 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[22256, 12648, 5367, 6879, 356, 2606, 1325, 14099] avg=8192.0000 max=22256 max_rank=0 min=356 min_rank=4 imbalance_ratio=2.716797 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[9094, 9811, 9421, 6202, 9811, 3549, 9811, 7837] avg=8192.0000 max=9811 max_rank=1 min=3549 min_rank=5 imbalance_ratio=1.197632 max_load_reduction=7817 max_load_reduction_pct=0.443442 moved_to_replicas=29257 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[22256, 12648, 5367, 6879, 356, 2606, 1325, 14099] avg=8192.0000 max=22256 max_rank=0 min=356 min_rank=4 imbalance_ratio=2.716797 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[22256, 12648, 5367, 6879, 356, 2606, 1325, 14099] avg=8192.0000 max=22256 max_rank=0 min=356 min_rank=4 imbalance_ratio=2.716797 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[9094, 17628, 1604, 6202, 2647, 3549, 16975, 7837] avg=8192.0000 max=17628 max_rank=1 min=1604 min_rank=2 imbalance_ratio=2.151855 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[9094, 17628, 1604, 6202, 2647, 3549, 16975, 7837] avg=8192.0000 max=17628 max_rank=1 min=1604 min_rank=2 imbalance_ratio=2.151855 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[8496, 8497, 8496, 7901, 8498, 8207, 6943, 8498] avg=8192.0000 max=8498 max_rank=4 min=6943 min_rank=6 imbalance_ratio=1.037354 max_load_reduction=13758 max_load_reduction_pct=0.618170 moved_to_replicas=43065 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[22256, 12648, 5367, 6879, 356, 2606, 1325, 14099] avg=8192.0000 max=22256 max_rank=0 min=356 min_rank=4 imbalance_ratio=2.716797 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[9094, 17628, 1604, 6202, 2647, 3549, 16975, 7837] avg=8192.0000 max=17628 max_rank=1 min=1604 min_rank=2 imbalance_ratio=2.151855 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[8496, 8497, 8496, 7901, 8498, 8207, 6943, 8498] avg=8192.0000 max=8498 max_rank=4 min=6943 min_rank=6 imbalance_ratio=1.037354 max_load_reduction=13758 max_load_reduction_pct=0.618170 moved_to_replicas=43065 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[22256, 12648, 5367, 6879, 356, 2606, 1325, 14099] avg=8192.0000 max=22256 max_rank=0 min=356 min_rank=4 imbalance_ratio=2.716797 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[8496, 8497, 8496, 7901, 8498, 8207, 6943, 8498] avg=8192.0000 max=8498 max_rank=4 min=6943 min_rank=6 imbalance_ratio=1.037354 max_load_reduction=13758 max_load_reduction_pct=0.618170 moved_to_replicas=43065 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[9094, 9811, 9421, 6202, 9811, 3549, 9811, 7837] avg=8192.0000 max=9811 max_rank=1 min=3549 min_rank=5 imbalance_ratio=1.197632 max_load_reduction=7817 max_load_reduction_pct=0.443442 moved_to_replicas=29257 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[9094, 9811, 9421, 6202, 9811, 3549, 9811, 7837] avg=8192.0000 max=9811 max_rank=1 min=3549 min_rank=5 imbalance_ratio=1.197632 max_load_reduction=7817 max_load_reduction_pct=0.443442 moved_to_replicas=29257 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[8496, 8497, 8496, 7901, 8498, 8207, 6943, 8498] avg=8192.0000 max=8498 max_rank=4 min=6943 min_rank=6 imbalance_ratio=1.037354 max_load_reduction=13758 max_load_reduction_pct=0.618170 moved_to_replicas=43065 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[9094, 9811, 9421, 6202, 9811, 3549, 9811, 7837] avg=8192.0000 max=9811 max_rank=1 min=3549 min_rank=5 imbalance_ratio=1.197632 max_load_reduction=7817 max_load_reduction_pct=0.443442 moved_to_replicas=29257 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[14929, 2443, 4799, 17547, 3696, 11511, 9223, 1388] avg=8192.0000 max=17547 max_rank=3 min=1388 min_rank=7 imbalance_ratio=2.141968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[8496, 8497, 8496, 7901, 8498, 8207, 6943, 8498] avg=8192.0000 max=8498 max_rank=4 min=6943 min_rank=6 imbalance_ratio=1.037354 max_load_reduction=13758 max_load_reduction_pct=0.618170 moved_to_replicas=43065 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[14929, 2443, 4799, 17547, 3696, 11511, 9223, 1388] avg=8192.0000 max=17547 max_rank=3 min=1388 min_rank=7 imbalance_ratio=2.141968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[9469, 7903, 4799, 9467, 5739, 9468, 9223, 9468] avg=8192.0000 max=9469 max_rank=0 min=4799 min_rank=2 imbalance_ratio=1.155884 max_load_reduction=8078 max_load_reduction_pct=0.460364 moved_to_replicas=25167 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[14929, 2443, 4799, 17547, 3696, 11511, 9223, 1388] avg=8192.0000 max=17547 max_rank=3 min=1388 min_rank=7 imbalance_ratio=2.141968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[22256, 12648, 5367, 6879, 356, 2606, 1325, 14099] avg=8192.0000 max=22256 max_rank=0 min=356 min_rank=4 imbalance_ratio=2.716797 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[14929, 2443, 4799, 17547, 3696, 11511, 9223, 1388] avg=8192.0000 max=17547 max_rank=3 min=1388 min_rank=7 imbalance_ratio=2.141968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[22256, 12648, 5367, 6879, 356, 2606, 1325, 14099] avg=8192.0000 max=22256 max_rank=0 min=356 min_rank=4 imbalance_ratio=2.716797 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[9469, 7903, 4799, 9467, 5739, 9468, 9223, 9468] avg=8192.0000 max=9469 max_rank=0 min=4799 min_rank=2 imbalance_ratio=1.155884 max_load_reduction=8078 max_load_reduction_pct=0.460364 moved_to_replicas=25167 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[14929, 2443, 4799, 17547, 3696, 11511, 9223, 1388] avg=8192.0000 max=17547 max_rank=3 min=1388 min_rank=7 imbalance_ratio=2.141968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[22256, 12648, 5367, 6879, 356, 2606, 1325, 14099] avg=8192.0000 max=22256 max_rank=0 min=356 min_rank=4 imbalance_ratio=2.716797 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[9469, 7903, 4799, 9467, 5739, 9468, 9223, 9468] avg=8192.0000 max=9469 max_rank=0 min=4799 min_rank=2 imbalance_ratio=1.155884 max_load_reduction=8078 max_load_reduction_pct=0.460364 moved_to_replicas=25167 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[8496, 8497, 8496, 7901, 8498, 8207, 6943, 8498] avg=8192.0000 max=8498 max_rank=4 min=6943 min_rank=6 imbalance_ratio=1.037354 max_load_reduction=13758 max_load_reduction_pct=0.618170 moved_to_replicas=43065 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[9469, 7903, 4799, 9467, 5739, 9468, 9223, 9468] avg=8192.0000 max=9469 max_rank=0 min=4799 min_rank=2 imbalance_ratio=1.155884 max_load_reduction=8078 max_load_reduction_pct=0.460364 moved_to_replicas=25167 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[9360, 12070, 4842, 18850, 5183, 2776, 1980, 10475] avg=8192.0000 max=18850 max_rank=3 min=1980 min_rank=6 imbalance_ratio=2.301025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[8496, 8497, 8496, 7901, 8498, 8207, 6943, 8498] avg=8192.0000 max=8498 max_rank=4 min=6943 min_rank=6 imbalance_ratio=1.037354 max_load_reduction=13758 max_load_reduction_pct=0.618170 moved_to_replicas=43065 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[9469, 7903, 4799, 9467, 5739, 9468, 9223, 9468] avg=8192.0000 max=9469 max_rank=0 min=4799 min_rank=2 imbalance_ratio=1.155884 max_load_reduction=8078 max_load_reduction_pct=0.460364 moved_to_replicas=25167 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[8496, 8497, 8496, 7901, 8498, 8207, 6943, 8498] avg=8192.0000 max=8498 max_rank=4 min=6943 min_rank=6 imbalance_ratio=1.037354 max_load_reduction=13758 max_load_reduction_pct=0.618170 moved_to_replicas=43065 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[9360, 12070, 4842, 18850, 5183, 2776, 1980, 10475] avg=8192.0000 max=18850 max_rank=3 min=1980 min_rank=6 imbalance_ratio=2.301025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[9360, 10475, 4842, 10475, 5183, 4371, 10355, 10475] avg=8192.0000 max=10475 max_rank=1 min=4371 min_rank=5 imbalance_ratio=1.278687 max_load_reduction=8375 max_load_reduction_pct=0.444297 moved_to_replicas=24673 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[9360, 12070, 4842, 18850, 5183, 2776, 1980, 10475] avg=8192.0000 max=18850 max_rank=3 min=1980 min_rank=6 imbalance_ratio=2.301025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[9360, 10475, 4842, 10475, 5183, 4371, 10355, 10475] avg=8192.0000 max=10475 max_rank=1 min=4371 min_rank=5 imbalance_ratio=1.278687 max_load_reduction=8375 max_load_reduction_pct=0.444297 moved_to_replicas=24673 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[14929, 2443, 4799, 17547, 3696, 11511, 9223, 1388] avg=8192.0000 max=17547 max_rank=3 min=1388 min_rank=7 imbalance_ratio=2.141968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[9360, 12070, 4842, 18850, 5183, 2776, 1980, 10475] avg=8192.0000 max=18850 max_rank=3 min=1980 min_rank=6 imbalance_ratio=2.301025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[9360, 12070, 4842, 18850, 5183, 2776, 1980, 10475] avg=8192.0000 max=18850 max_rank=3 min=1980 min_rank=6 imbalance_ratio=2.301025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[14929, 2443, 4799, 17547, 3696, 11511, 9223, 1388] avg=8192.0000 max=17547 max_rank=3 min=1388 min_rank=7 imbalance_ratio=2.141968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[14929, 2443, 4799, 17547, 3696, 11511, 9223, 1388] avg=8192.0000 max=17547 max_rank=3 min=1388 min_rank=7 imbalance_ratio=2.141968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[9360, 10475, 4842, 10475, 5183, 4371, 10355, 10475] avg=8192.0000 max=10475 max_rank=1 min=4371 min_rank=5 imbalance_ratio=1.278687 max_load_reduction=8375 max_load_reduction_pct=0.444297 moved_to_replicas=24673 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[9360, 10475, 4842, 10475, 5183, 4371, 10355, 10475] avg=8192.0000 max=10475 max_rank=1 min=4371 min_rank=5 imbalance_ratio=1.278687 max_load_reduction=8375 max_load_reduction_pct=0.444297 moved_to_replicas=24673 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[9469, 7903, 4799, 9467, 5739, 9468, 9223, 9468] avg=8192.0000 max=9469 max_rank=0 min=4799 min_rank=2 imbalance_ratio=1.155884 max_load_reduction=8078 max_load_reduction_pct=0.460364 moved_to_replicas=25167 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[2470, 2589, 11667, 16448, 12590, 2959, 5133, 11680] avg=8192.0000 max=16448 max_rank=3 min=2470 min_rank=0 imbalance_ratio=2.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[2470, 2589, 11667, 16448, 12590, 2959, 5133, 11680] avg=8192.0000 max=16448 max_rank=3 min=2470 min_rank=0 imbalance_ratio=2.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[9360, 10475, 4842, 10475, 5183, 4371, 10355, 10475] avg=8192.0000 max=10475 max_rank=1 min=4371 min_rank=5 imbalance_ratio=1.278687 max_load_reduction=8375 max_load_reduction_pct=0.444297 moved_to_replicas=24673 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[9469, 7903, 4799, 9467, 5739, 9468, 9223, 9468] avg=8192.0000 max=9469 max_rank=0 min=4799 min_rank=2 imbalance_ratio=1.155884 max_load_reduction=8078 max_load_reduction_pct=0.460364 moved_to_replicas=25167 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[9469, 7903, 4799, 9467, 5739, 9468, 9223, 9468] avg=8192.0000 max=9469 max_rank=0 min=4799 min_rank=2 imbalance_ratio=1.155884 max_load_reduction=8078 max_load_reduction_pct=0.460364 moved_to_replicas=25167 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[9460, 5720, 9460, 9458, 9459, 5180, 7340, 9459] avg=8192.0000 max=9460 max_rank=0 min=5180 min_rank=5 imbalance_ratio=1.154785 max_load_reduction=6988 max_load_reduction_pct=0.424854 moved_to_replicas=20387 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[2470, 2589, 11667, 16448, 12590, 2959, 5133, 11680] avg=8192.0000 max=16448 max_rank=3 min=2470 min_rank=0 imbalance_ratio=2.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[9460, 5720, 9460, 9458, 9459, 5180, 7340, 9459] avg=8192.0000 max=9460 max_rank=0 min=5180 min_rank=5 imbalance_ratio=1.154785 max_load_reduction=6988 max_load_reduction_pct=0.424854 moved_to_replicas=20387 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[2470, 2589, 11667, 16448, 12590, 2959, 5133, 11680] avg=8192.0000 max=16448 max_rank=3 min=2470 min_rank=0 imbalance_ratio=2.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[9360, 12070, 4842, 18850, 5183, 2776, 1980, 10475] avg=8192.0000 max=18850 max_rank=3 min=1980 min_rank=6 imbalance_ratio=2.301025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[2470, 2589, 11667, 16448, 12590, 2959, 5133, 11680] avg=8192.0000 max=16448 max_rank=3 min=2470 min_rank=0 imbalance_ratio=2.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[9360, 12070, 4842, 18850, 5183, 2776, 1980, 10475] avg=8192.0000 max=18850 max_rank=3 min=1980 min_rank=6 imbalance_ratio=2.301025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[9360, 12070, 4842, 18850, 5183, 2776, 1980, 10475] avg=8192.0000 max=18850 max_rank=3 min=1980 min_rank=6 imbalance_ratio=2.301025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[9460, 5720, 9460, 9458, 9459, 5180, 7340, 9459] avg=8192.0000 max=9460 max_rank=0 min=5180 min_rank=5 imbalance_ratio=1.154785 max_load_reduction=6988 max_load_reduction_pct=0.424854 moved_to_replicas=20387 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[9460, 5720, 9460, 9458, 9459, 5180, 7340, 9459] avg=8192.0000 max=9460 max_rank=0 min=5180 min_rank=5 imbalance_ratio=1.154785 max_load_reduction=6988 max_load_reduction_pct=0.424854 moved_to_replicas=20387 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[1260, 7241, 7334, 11876, 3378, 17248, 5482, 11717] avg=8192.0000 max=17248 max_rank=5 min=1260 min_rank=0 imbalance_ratio=2.105469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[9360, 10475, 4842, 10475, 5183, 4371, 10355, 10475] avg=8192.0000 max=10475 max_rank=1 min=4371 min_rank=5 imbalance_ratio=1.278687 max_load_reduction=8375 max_load_reduction_pct=0.444297 moved_to_replicas=24673 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[1260, 7241, 7334, 11876, 3378, 17248, 5482, 11717] avg=8192.0000 max=17248 max_rank=5 min=1260 min_rank=0 imbalance_ratio=2.105469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[9460, 5720, 9460, 9458, 9459, 5180, 7340, 9459] avg=8192.0000 max=9460 max_rank=0 min=5180 min_rank=5 imbalance_ratio=1.154785 max_load_reduction=6988 max_load_reduction_pct=0.424854 moved_to_replicas=20387 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[9360, 10475, 4842, 10475, 5183, 4371, 10355, 10475] avg=8192.0000 max=10475 max_rank=1 min=4371 min_rank=5 imbalance_ratio=1.278687 max_load_reduction=8375 max_load_reduction_pct=0.444297 moved_to_replicas=24673 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[9360, 10475, 4842, 10475, 5183, 4371, 10355, 10475] avg=8192.0000 max=10475 max_rank=1 min=4371 min_rank=5 imbalance_ratio=1.278687 max_load_reduction=8375 max_load_reduction_pct=0.444297 moved_to_replicas=24673 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[9253, 7241, 7334, 9254, 6000, 9255, 7944, 9255] avg=8192.0000 max=9255 max_rank=5 min=6000 min_rank=4 imbalance_ratio=1.129761 max_load_reduction=7993 max_load_reduction_pct=0.463416 moved_to_replicas=25228 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[1260, 7241, 7334, 11876, 3378, 17248, 5482, 11717] avg=8192.0000 max=17248 max_rank=5 min=1260 min_rank=0 imbalance_ratio=2.105469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[9253, 7241, 7334, 9254, 6000, 9255, 7944, 9255] avg=8192.0000 max=9255 max_rank=5 min=6000 min_rank=4 imbalance_ratio=1.129761 max_load_reduction=7993 max_load_reduction_pct=0.463416 moved_to_replicas=25228 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[1260, 7241, 7334, 11876, 3378, 17248, 5482, 11717] avg=8192.0000 max=17248 max_rank=5 min=1260 min_rank=0 imbalance_ratio=2.105469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[2470, 2589, 11667, 16448, 12590, 2959, 5133, 11680] avg=8192.0000 max=16448 max_rank=3 min=2470 min_rank=0 imbalance_ratio=2.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[1260, 7241, 7334, 11876, 3378, 17248, 5482, 11717] avg=8192.0000 max=17248 max_rank=5 min=1260 min_rank=0 imbalance_ratio=2.105469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[9253, 7241, 7334, 9254, 6000, 9255, 7944, 9255] avg=8192.0000 max=9255 max_rank=5 min=6000 min_rank=4 imbalance_ratio=1.129761 max_load_reduction=7993 max_load_reduction_pct=0.463416 moved_to_replicas=25228 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[2470, 2589, 11667, 16448, 12590, 2959, 5133, 11680] avg=8192.0000 max=16448 max_rank=3 min=2470 min_rank=0 imbalance_ratio=2.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[2470, 2589, 11667, 16448, 12590, 2959, 5133, 11680] avg=8192.0000 max=16448 max_rank=3 min=2470 min_rank=0 imbalance_ratio=2.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[9253, 7241, 7334, 9254, 6000, 9255, 7944, 9255] avg=8192.0000 max=9255 max_rank=5 min=6000 min_rank=4 imbalance_ratio=1.129761 max_load_reduction=7993 max_load_reduction_pct=0.463416 moved_to_replicas=25228 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[5752, 15620, 6294, 904, 5460, 17674, 3022, 10810] avg=8192.0000 max=17674 max_rank=5 min=904 min_rank=3 imbalance_ratio=2.157471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[9460, 5720, 9460, 9458, 9459, 5180, 7340, 9459] avg=8192.0000 max=9460 max_rank=0 min=5180 min_rank=5 imbalance_ratio=1.154785 max_load_reduction=6988 max_load_reduction_pct=0.424854 moved_to_replicas=20387 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[5752, 15620, 6294, 904, 5460, 17674, 3022, 10810] avg=8192.0000 max=17674 max_rank=5 min=904 min_rank=3 imbalance_ratio=2.157471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[9253, 7241, 7334, 9254, 6000, 9255, 7944, 9255] avg=8192.0000 max=9255 max_rank=5 min=6000 min_rank=4 imbalance_ratio=1.129761 max_load_reduction=7993 max_load_reduction_pct=0.463416 moved_to_replicas=25228 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[9460, 5720, 9460, 9458, 9459, 5180, 7340, 9459] avg=8192.0000 max=9460 max_rank=0 min=5180 min_rank=5 imbalance_ratio=1.154785 max_load_reduction=6988 max_load_reduction_pct=0.424854 moved_to_replicas=20387 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[9460, 5720, 9460, 9458, 9459, 5180, 7340, 9459] avg=8192.0000 max=9460 max_rank=0 min=5180 min_rank=5 imbalance_ratio=1.154785 max_load_reduction=6988 max_load_reduction_pct=0.424854 moved_to_replicas=20387 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[5752, 10810, 6294, 7767, 5460, 10811, 7832, 10810] avg=8192.0000 max=10811 max_rank=5 min=5460 min_rank=4 imbalance_ratio=1.319702 max_load_reduction=6863 max_load_reduction_pct=0.388311 moved_to_replicas=23206 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[5752, 15620, 6294, 904, 5460, 17674, 3022, 10810] avg=8192.0000 max=17674 max_rank=5 min=904 min_rank=3 imbalance_ratio=2.157471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[5752, 10810, 6294, 7767, 5460, 10811, 7832, 10810] avg=8192.0000 max=10811 max_rank=5 min=5460 min_rank=4 imbalance_ratio=1.319702 max_load_reduction=6863 max_load_reduction_pct=0.388311 moved_to_replicas=23206 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[5752, 15620, 6294, 904, 5460, 17674, 3022, 10810] avg=8192.0000 max=17674 max_rank=5 min=904 min_rank=3 imbalance_ratio=2.157471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[1260, 7241, 7334, 11876, 3378, 17248, 5482, 11717] avg=8192.0000 max=17248 max_rank=5 min=1260 min_rank=0 imbalance_ratio=2.105469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[5752, 15620, 6294, 904, 5460, 17674, 3022, 10810] avg=8192.0000 max=17674 max_rank=5 min=904 min_rank=3 imbalance_ratio=2.157471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[5752, 10810, 6294, 7767, 5460, 10811, 7832, 10810] avg=8192.0000 max=10811 max_rank=5 min=5460 min_rank=4 imbalance_ratio=1.319702 max_load_reduction=6863 max_load_reduction_pct=0.388311 moved_to_replicas=23206 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[1260, 7241, 7334, 11876, 3378, 17248, 5482, 11717] avg=8192.0000 max=17248 max_rank=5 min=1260 min_rank=0 imbalance_ratio=2.105469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[1260, 7241, 7334, 11876, 3378, 17248, 5482, 11717] avg=8192.0000 max=17248 max_rank=5 min=1260 min_rank=0 imbalance_ratio=2.105469 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[5752, 10810, 6294, 7767, 5460, 10811, 7832, 10810] avg=8192.0000 max=10811 max_rank=5 min=5460 min_rank=4 imbalance_ratio=1.319702 max_load_reduction=6863 max_load_reduction_pct=0.388311 moved_to_replicas=23206 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[9253, 7241, 7334, 9254, 6000, 9255, 7944, 9255] avg=8192.0000 max=9255 max_rank=5 min=6000 min_rank=4 imbalance_ratio=1.129761 max_load_reduction=7993 max_load_reduction_pct=0.463416 moved_to_replicas=25228 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[5752, 10810, 6294, 7767, 5460, 10811, 7832, 10810] avg=8192.0000 max=10811 max_rank=5 min=5460 min_rank=4 imbalance_ratio=1.319702 max_load_reduction=6863 max_load_reduction_pct=0.388311 moved_to_replicas=23206 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[9253, 7241, 7334, 9254, 6000, 9255, 7944, 9255] avg=8192.0000 max=9255 max_rank=5 min=6000 min_rank=4 imbalance_ratio=1.129761 max_load_reduction=7993 max_load_reduction_pct=0.463416 moved_to_replicas=25228 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[9253, 7241, 7334, 9254, 6000, 9255, 7944, 9255] avg=8192.0000 max=9255 max_rank=5 min=6000 min_rank=4 imbalance_ratio=1.129761 max_load_reduction=7993 max_load_reduction_pct=0.463416 moved_to_replicas=25228 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[5752, 15620, 6294, 904, 5460, 17674, 3022, 10810] avg=8192.0000 max=17674 max_rank=5 min=904 min_rank=3 imbalance_ratio=2.157471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[5752, 15620, 6294, 904, 5460, 17674, 3022, 10810] avg=8192.0000 max=17674 max_rank=5 min=904 min_rank=3 imbalance_ratio=2.157471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[5752, 15620, 6294, 904, 5460, 17674, 3022, 10810] avg=8192.0000 max=17674 max_rank=5 min=904 min_rank=3 imbalance_ratio=2.157471 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[5752, 10810, 6294, 7767, 5460, 10811, 7832, 10810] avg=8192.0000 max=10811 max_rank=5 min=5460 min_rank=4 imbalance_ratio=1.319702 max_load_reduction=6863 max_load_reduction_pct=0.388311 moved_to_replicas=23206 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[5752, 10810, 6294, 7767, 5460, 10811, 7832, 10810] avg=8192.0000 max=10811 max_rank=5 min=5460 min_rank=4 imbalance_ratio=1.319702 max_load_reduction=6863 max_load_reduction_pct=0.388311 moved_to_replicas=23206 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[5752, 10810, 6294, 7767, 5460, 10811, 7832, 10810] avg=8192.0000 max=10811 max_rank=5 min=5460 min_rank=4 imbalance_ratio=1.319702 max_load_reduction=6863 max_load_reduction_pct=0.388311 moved_to_replicas=23206 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=32 total_expected_recv=247286 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=224768 peak_demand=10747 required_capacity=1.311890 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=32 total_expected_recv=269789 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=202624 peak_demand=10987 required_capacity=1.341187 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=32 total_expected_recv=260433 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=212096 peak_demand=12556 required_capacity=1.532715 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=32 total_expected_recv=257733 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=214400 peak_demand=10872 required_capacity=1.327148 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=32 total_expected_recv=277391 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=194816 peak_demand=12556 required_capacity=1.532715 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=32 total_expected_recv=265965 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=206464 peak_demand=9868 required_capacity=1.204590 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=32 total_expected_recv=270210 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=201344 peak_demand=12413 required_capacity=1.515259 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=32 total_expected_recv=248345 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=223232 peak_demand=10872 required_capacity=1.327148 + [2026-08-30 05:09:11] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 855.3 | throughput per GPU (TFLOP/s/GPU): 389.8 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.104935E+01 | loss scale: 1.0 | grad norm: 37.624 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 13312 | min safe capacity: 1.6250 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=14848 expected_recv=16421 expected_overflow=0 exact_recv_per_rank=[7300, 8625, 8995, 7651, 8995, 5980, 8996, 8994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=14848 expected_recv=17582 expected_overflow=0 exact_recv_per_rank=[7300, 8625, 8995, 7651, 8995, 5980, 8996, 8994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=14848 expected_recv=16174 expected_overflow=0 exact_recv_per_rank=[7300, 8625, 8995, 7651, 8995, 5980, 8996, 8994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=14848 expected_recv=13247 expected_overflow=0 exact_recv_per_rank=[7300, 8625, 8995, 7651, 8995, 5980, 8996, 8994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=14848 expected_recv=17580 expected_overflow=0 exact_recv_per_rank=[7300, 8625, 8995, 7651, 8995, 5980, 8996, 8994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=14848 expected_recv=17581 expected_overflow=0 exact_recv_per_rank=[7300, 8625, 8995, 7651, 8995, 5980, 8996, 8994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=14848 expected_recv=14907 expected_overflow=0 exact_recv_per_rank=[7300, 8625, 8995, 7651, 8995, 5980, 8996, 8994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=14848 expected_recv=16421 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=14848 expected_recv=17582 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=14848 expected_recv=16174 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=14848 expected_recv=13247 actual_valid_recv=13440 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=14848 expected_recv=17580 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17792 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=14848 expected_recv=17580 expected_overflow=0 exact_recv_per_rank=[7300, 8625, 8995, 7651, 8995, 5980, 8996, 8994] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=14848 expected_recv=17581 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=13440 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=14848 expected_recv=14907 actual_valid_recv=15232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17792 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17792 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=15232 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=14848 expected_recv=17580 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17792 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=14848 expected_recv=18147 expected_overflow=0 exact_recv_per_rank=[8879, 8878, 7927, 8879, 8444, 8878, 8878, 4773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=14848 expected_recv=17194 expected_overflow=0 exact_recv_per_rank=[8879, 8878, 7927, 8879, 8444, 8878, 8878, 4773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=14848 expected_recv=13337 expected_overflow=0 exact_recv_per_rank=[8879, 8878, 7927, 8879, 8444, 8878, 8878, 4773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=14848 expected_recv=18147 actual_valid_recv=18304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=14848 expected_recv=10242 expected_overflow=0 exact_recv_per_rank=[8879, 8878, 7927, 8879, 8444, 8878, 8878, 4773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=14848 expected_recv=18146 expected_overflow=0 exact_recv_per_rank=[8879, 8878, 7927, 8879, 8444, 8878, 8878, 4773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=14848 expected_recv=17194 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=14848 expected_recv=18148 expected_overflow=0 exact_recv_per_rank=[8879, 8878, 7927, 8879, 8444, 8878, 8878, 4773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=14848 expected_recv=18148 expected_overflow=0 exact_recv_per_rank=[8879, 8878, 7927, 8879, 8444, 8878, 8878, 4773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=18304 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=14848 expected_recv=13337 actual_valid_recv=13568 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=14848 expected_recv=10242 actual_valid_recv=10496 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=14848 expected_recv=18146 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=17408 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=14848 expected_recv=18148 actual_valid_recv=18304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=10496 slack_slots=19200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=14848 expected_recv=18148 actual_valid_recv=18304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=13568 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=18560 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=14848 expected_recv=17710 expected_overflow=0 exact_recv_per_rank=[8879, 8878, 7927, 8879, 8444, 8878, 8878, 4773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=18304 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=18304 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=14848 expected_recv=17710 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=17920 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=14848 expected_recv=11454 expected_overflow=0 exact_recv_per_rank=[4937, 8879, 8879, 7325, 8879, 8879, 8879, 8879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=14848 expected_recv=19670 expected_overflow=0 exact_recv_per_rank=[4937, 8879, 8879, 7325, 8879, 8879, 8879, 8879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=14848 expected_recv=19669 expected_overflow=0 exact_recv_per_rank=[4937, 8879, 8879, 7325, 8879, 8879, 8879, 8879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=14848 expected_recv=19363 expected_overflow=0 exact_recv_per_rank=[4937, 8879, 8879, 7325, 8879, 8879, 8879, 8879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=14848 expected_recv=19670 expected_overflow=0 exact_recv_per_rank=[4937, 8879, 8879, 7325, 8879, 8879, 8879, 8879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=14848 expected_recv=11454 actual_valid_recv=11776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=14848 expected_recv=19670 actual_valid_recv=19968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=14848 expected_recv=19669 actual_valid_recv=19840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=14848 expected_recv=19363 actual_valid_recv=19456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=14848 expected_recv=10189 expected_overflow=0 exact_recv_per_rank=[4937, 8879, 8879, 7325, 8879, 8879, 8879, 8879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=14848 expected_recv=19670 actual_valid_recv=19840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=11776 slack_slots=17920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=14848 expected_recv=11387 expected_overflow=0 exact_recv_per_rank=[4937, 8879, 8879, 7325, 8879, 8879, 8879, 8879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=19968 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=19456 slack_slots=10240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=19840 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=19840 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=14848 expected_recv=10189 actual_valid_recv=10368 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=14848 expected_recv=11387 actual_valid_recv=11520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=14848 expected_recv=19670 expected_overflow=0 exact_recv_per_rank=[4937, 8879, 8879, 7325, 8879, 8879, 8879, 8879] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=10368 slack_slots=19328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=11520 slack_slots=18176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=14848 expected_recv=19670 actual_valid_recv=19840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=19840 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=14848 expected_recv=17332 expected_overflow=0 exact_recv_per_rank=[7783, 8765, 8211, 8522, 5957, 8766, 8766, 8766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=14848 expected_recv=16779 expected_overflow=0 exact_recv_per_rank=[7783, 8765, 8211, 8522, 5957, 8766, 8766, 8766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=14848 expected_recv=17334 expected_overflow=0 exact_recv_per_rank=[7783, 8765, 8211, 8522, 5957, 8766, 8766, 8766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=14848 expected_recv=15342 expected_overflow=0 exact_recv_per_rank=[7783, 8765, 8211, 8522, 5957, 8766, 8766, 8766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=14848 expected_recv=17334 expected_overflow=0 exact_recv_per_rank=[7783, 8765, 8211, 8522, 5957, 8766, 8766, 8766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=14848 expected_recv=17332 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=14848 expected_recv=16779 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=14848 expected_recv=17334 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=14848 expected_recv=15342 actual_valid_recv=15616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=14848 expected_recv=17091 expected_overflow=0 exact_recv_per_rank=[7783, 8765, 8211, 8522, 5957, 8766, 8766, 8766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=14848 expected_recv=17334 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=14848 expected_recv=17091 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=17536 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=14848 expected_recv=15796 expected_overflow=0 exact_recv_per_rank=[7783, 8765, 8211, 8522, 5957, 8766, 8766, 8766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=15616 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=17408 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=14848 expected_recv=14064 expected_overflow=0 exact_recv_per_rank=[7783, 8765, 8211, 8522, 5957, 8766, 8766, 8766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=17408 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=17280 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=14848 expected_recv=15796 actual_valid_recv=15872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=14848 expected_recv=14064 actual_valid_recv=14336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=15872 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=14336 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=14848 expected_recv=17055 expected_overflow=0 exact_recv_per_rank=[6720, 8090, 8400, 8523, 8522, 8523, 8523, 8235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=14848 expected_recv=17366 expected_overflow=0 exact_recv_per_rank=[6720, 8090, 8400, 8523, 8522, 8523, 8523, 8235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=14848 expected_recv=11009 expected_overflow=0 exact_recv_per_rank=[6720, 8090, 8400, 8523, 8522, 8523, 8523, 8235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=14848 expected_recv=17490 expected_overflow=0 exact_recv_per_rank=[6720, 8090, 8400, 8523, 8522, 8523, 8523, 8235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=14848 expected_recv=17489 expected_overflow=0 exact_recv_per_rank=[6720, 8090, 8400, 8523, 8522, 8523, 8523, 8235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=14848 expected_recv=17488 expected_overflow=0 exact_recv_per_rank=[6720, 8090, 8400, 8523, 8522, 8523, 8523, 8235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=14848 expected_recv=17055 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=14848 expected_recv=17366 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=14848 expected_recv=11009 actual_valid_recv=11136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=14848 expected_recv=17490 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=14848 expected_recv=17489 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=14848 expected_recv=17488 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=17280 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=17536 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=11136 slack_slots=18560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=17664 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=17664 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=14848 expected_recv=17488 expected_overflow=0 exact_recv_per_rank=[6720, 8090, 8400, 8523, 8522, 8523, 8523, 8235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=17664 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=14848 expected_recv=15687 expected_overflow=0 exact_recv_per_rank=[6720, 8090, 8400, 8523, 8522, 8523, 8523, 8235] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=14848 expected_recv=17488 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=14848 expected_recv=15687 actual_valid_recv=16000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=17920 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16000 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=14848 expected_recv=16871 expected_overflow=0 exact_recv_per_rank=[8595, 8596, 8595, 7416, 8594, 8595, 6550, 8595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=14848 expected_recv=16869 expected_overflow=0 exact_recv_per_rank=[8595, 8596, 8595, 7416, 8594, 8595, 6550, 8595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=14848 expected_recv=16869 expected_overflow=0 exact_recv_per_rank=[8595, 8596, 8595, 7416, 8594, 8595, 6550, 8595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=14848 expected_recv=14464 expected_overflow=0 exact_recv_per_rank=[8595, 8596, 8595, 7416, 8594, 8595, 6550, 8595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=14848 expected_recv=15394 expected_overflow=0 exact_recv_per_rank=[8595, 8596, 8595, 7416, 8594, 8595, 6550, 8595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=14848 expected_recv=16871 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=14848 expected_recv=16869 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=14848 expected_recv=16868 expected_overflow=0 exact_recv_per_rank=[8595, 8596, 8595, 7416, 8594, 8595, 6550, 8595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=14848 expected_recv=16869 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=14848 expected_recv=14464 actual_valid_recv=14720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=14848 expected_recv=15394 actual_valid_recv=15744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=17152 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=14848 expected_recv=16868 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=14848 expected_recv=16867 expected_overflow=0 exact_recv_per_rank=[8595, 8596, 8595, 7416, 8594, 8595, 6550, 8595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=17152 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=14720 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=15744 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=14848 expected_recv=16870 expected_overflow=0 exact_recv_per_rank=[8595, 8596, 8595, 7416, 8594, 8595, 6550, 8595] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=14848 expected_recv=16867 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=14848 expected_recv=16870 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=17152 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=14848 expected_recv=18538 expected_overflow=0 exact_recv_per_rank=[9284, 7779, 9282, 9283, 9282, 9282, 2061, 9283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=14848 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[9284, 7779, 9282, 9283, 9282, 9282, 2061, 9283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=14848 expected_recv=18537 expected_overflow=0 exact_recv_per_rank=[9284, 7779, 9282, 9283, 9282, 9282, 2061, 9283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=14848 expected_recv=11188 expected_overflow=0 exact_recv_per_rank=[9284, 7779, 9282, 9283, 9282, 9282, 2061, 9283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=14848 expected_recv=11315 expected_overflow=0 exact_recv_per_rank=[9284, 7779, 9282, 9283, 9282, 9282, 2061, 9283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=14848 expected_recv=18538 actual_valid_recv=18688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=14848 expected_recv=18535 expected_overflow=0 exact_recv_per_rank=[9284, 7779, 9282, 9283, 9282, 9282, 2061, 9283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=14848 expected_recv=15888 actual_valid_recv=16000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=14848 expected_recv=18537 actual_valid_recv=18688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=14848 expected_recv=11188 actual_valid_recv=11520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=18688 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=14848 expected_recv=11315 actual_valid_recv=11648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16000 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=18688 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=14848 expected_recv=18535 actual_valid_recv=18816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=11520 slack_slots=18176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=14848 expected_recv=18533 expected_overflow=0 exact_recv_per_rank=[9284, 7779, 9282, 9283, 9282, 9282, 2061, 9283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=11648 slack_slots=18048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=18816 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=14848 expected_recv=18538 expected_overflow=0 exact_recv_per_rank=[9284, 7779, 9282, 9283, 9282, 9282, 2061, 9283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=14848 expected_recv=18533 actual_valid_recv=18688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=14848 expected_recv=18538 actual_valid_recv=18944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=18688 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=14848 expected_recv=17460 expected_overflow=0 exact_recv_per_rank=[8578, 8580, 8580, 8572, 8579, 8578, 5490, 8579] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=18944 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=14848 expected_recv=17462 expected_overflow=0 exact_recv_per_rank=[8578, 8580, 8580, 8572, 8579, 8578, 5490, 8579] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=14848 expected_recv=17462 expected_overflow=0 exact_recv_per_rank=[8578, 8580, 8580, 8572, 8579, 8578, 5490, 8579] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=14848 expected_recv=17460 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=14848 expected_recv=16970 expected_overflow=0 exact_recv_per_rank=[8578, 8580, 8580, 8572, 8579, 8578, 5490, 8579] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=14848 expected_recv=17462 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=14848 expected_recv=17462 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=17664 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=14848 expected_recv=14372 expected_overflow=0 exact_recv_per_rank=[8578, 8580, 8580, 8572, 8579, 8578, 5490, 8579] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=14848 expected_recv=16970 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=14848 expected_recv=17461 expected_overflow=0 exact_recv_per_rank=[8578, 8580, 8580, 8572, 8579, 8578, 5490, 8579] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=17664 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=17792 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=14848 expected_recv=14372 actual_valid_recv=14720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=17280 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=14848 expected_recv=17461 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=14848 expected_recv=17461 expected_overflow=0 exact_recv_per_rank=[8578, 8580, 8580, 8572, 8579, 8578, 5490, 8579] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=14848 expected_recv=12424 expected_overflow=0 exact_recv_per_rank=[8578, 8580, 8580, 8572, 8579, 8578, 5490, 8579] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=14720 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=17792 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=14848 expected_recv=17461 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=14848 expected_recv=12424 actual_valid_recv=12672 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=14848 expected_recv=18415 expected_overflow=0 exact_recv_per_rank=[5368, 9238, 9238, 4742, 9237, 9238, 9237, 9238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=17664 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=12672 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=14848 expected_recv=18416 expected_overflow=0 exact_recv_per_rank=[5368, 9238, 9238, 4742, 9237, 9238, 9237, 9238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=14848 expected_recv=18415 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=14848 expected_recv=18416 actual_valid_recv=18688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=18560 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=14848 expected_recv=18416 expected_overflow=0 exact_recv_per_rank=[5368, 9238, 9238, 4742, 9237, 9238, 9237, 9238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=14848 expected_recv=13919 expected_overflow=0 exact_recv_per_rank=[5368, 9238, 9238, 4742, 9237, 9238, 9237, 9238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=18688 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=14848 expected_recv=16293 expected_overflow=0 exact_recv_per_rank=[5368, 9238, 9238, 4742, 9237, 9238, 9237, 9238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=14848 expected_recv=18411 expected_overflow=0 exact_recv_per_rank=[5368, 9238, 9238, 4742, 9237, 9238, 9237, 9238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=14848 expected_recv=13919 actual_valid_recv=14208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=14848 expected_recv=18416 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=14848 expected_recv=16293 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=14848 expected_recv=18414 expected_overflow=0 exact_recv_per_rank=[5368, 9238, 9238, 4742, 9237, 9238, 9237, 9238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=14848 expected_recv=18411 actual_valid_recv=18688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=14208 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=18560 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=14848 expected_recv=8788 expected_overflow=0 exact_recv_per_rank=[5368, 9238, 9238, 4742, 9237, 9238, 9237, 9238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=14848 expected_recv=18414 actual_valid_recv=18560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=18688 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=14848 expected_recv=8788 actual_valid_recv=9088 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=14848 expected_recv=20082 expected_overflow=0 exact_recv_per_rank=[6973, 3461, 7475, 9526, 9525, 9526, 9525, 9525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=18560 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=9088 slack_slots=20608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=14848 expected_recv=20082 actual_valid_recv=20224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=20224 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=14848 expected_recv=20083 expected_overflow=0 exact_recv_per_rank=[6973, 3461, 7475, 9526, 9525, 9526, 9525, 9525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=14848 expected_recv=16422 expected_overflow=0 exact_recv_per_rank=[6973, 3461, 7475, 9526, 9525, 9526, 9525, 9525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=14848 expected_recv=7389 expected_overflow=0 exact_recv_per_rank=[6973, 3461, 7475, 9526, 9525, 9526, 9525, 9525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=14848 expected_recv=19812 expected_overflow=0 exact_recv_per_rank=[6973, 3461, 7475, 9526, 9525, 9526, 9525, 9525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=14848 expected_recv=9673 expected_overflow=0 exact_recv_per_rank=[6973, 3461, 7475, 9526, 9525, 9526, 9525, 9525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=14848 expected_recv=20083 actual_valid_recv=20224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=14848 expected_recv=16422 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=14848 expected_recv=20081 expected_overflow=0 exact_recv_per_rank=[6973, 3461, 7475, 9526, 9525, 9526, 9525, 9525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=14848 expected_recv=7389 actual_valid_recv=7552 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=14848 expected_recv=19812 actual_valid_recv=20096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=14848 expected_recv=9673 actual_valid_recv=9856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=20224 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=14848 expected_recv=17530 expected_overflow=0 exact_recv_per_rank=[6973, 3461, 7475, 9526, 9525, 9526, 9525, 9525] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=14848 expected_recv=20081 actual_valid_recv=20224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=9856 slack_slots=19840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=7552 slack_slots=22144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=20096 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=14848 expected_recv=17208 expected_overflow=0 exact_recv_per_rank=[8799, 8680, 4156, 8798, 8798, 8798, 8798, 8709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=20224 slack_slots=9472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=14848 expected_recv=17530 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=14848 expected_recv=17208 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=17920 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=17408 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=14848 expected_recv=16945 expected_overflow=0 exact_recv_per_rank=[8799, 8680, 4156, 8798, 8798, 8798, 8798, 8709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=14848 expected_recv=8587 expected_overflow=0 exact_recv_per_rank=[8799, 8680, 4156, 8798, 8798, 8798, 8798, 8709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=14848 expected_recv=17689 expected_overflow=0 exact_recv_per_rank=[8799, 8680, 4156, 8798, 8798, 8798, 8798, 8709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=14848 expected_recv=17572 expected_overflow=0 exact_recv_per_rank=[8799, 8680, 4156, 8798, 8798, 8798, 8798, 8709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=14848 expected_recv=17690 expected_overflow=0 exact_recv_per_rank=[8799, 8680, 4156, 8798, 8798, 8798, 8798, 8709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=14848 expected_recv=8587 actual_valid_recv=8704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=14848 expected_recv=16945 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=14848 expected_recv=17689 actual_valid_recv=18048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=14848 expected_recv=17690 expected_overflow=0 exact_recv_per_rank=[8799, 8680, 4156, 8798, 8798, 8798, 8798, 8709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=14848 expected_recv=17690 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=14848 expected_recv=17572 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=8704 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=18048 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=17792 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=14848 expected_recv=17690 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=14848 expected_recv=15342 expected_overflow=0 exact_recv_per_rank=[8904, 6335, 8904, 8904, 8447, 6234, 8904, 8904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=14848 expected_recv=17691 expected_overflow=0 exact_recv_per_rank=[8799, 8680, 4156, 8798, 8798, 8798, 8798, 8709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=17920 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=17792 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=14848 expected_recv=15342 actual_valid_recv=15616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=14848 expected_recv=17691 actual_valid_recv=18048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=15616 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=18048 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=14848 expected_recv=17498 expected_overflow=0 exact_recv_per_rank=[8904, 6335, 8904, 8904, 8447, 6234, 8904, 8904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=14848 expected_recv=17500 expected_overflow=0 exact_recv_per_rank=[8904, 6335, 8904, 8904, 8447, 6234, 8904, 8904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=14848 expected_recv=17498 expected_overflow=0 exact_recv_per_rank=[8904, 6335, 8904, 8904, 8447, 6234, 8904, 8904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=14848 expected_recv=14828 expected_overflow=0 exact_recv_per_rank=[8904, 6335, 8904, 8904, 8447, 6234, 8904, 8904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=14848 expected_recv=17498 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=14848 expected_recv=17500 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=14848 expected_recv=17498 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=14848 expected_recv=14828 actual_valid_recv=14976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=14848 expected_recv=14477 expected_overflow=0 exact_recv_per_rank=[8904, 6335, 8904, 8904, 8447, 6234, 8904, 8904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=17664 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=14848 expected_recv=16430 expected_overflow=0 exact_recv_per_rank=[8904, 6335, 8904, 8904, 8447, 6234, 8904, 8904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=17920 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=17664 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=14976 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=14848 expected_recv=14477 actual_valid_recv=14848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=14848 expected_recv=17090 expected_overflow=0 exact_recv_per_rank=[8950, 8950, 8949, 8950, 8951, 2886, 8950, 8950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=14848 expected_recv=16430 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=14848 expected_recv=17499 expected_overflow=0 exact_recv_per_rank=[8904, 6335, 8904, 8904, 8447, 6234, 8904, 8904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=14848 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=14848 expected_recv=17090 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=14848 expected_recv=17499 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=17280 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=17664 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=14848 expected_recv=11400 expected_overflow=0 exact_recv_per_rank=[8950, 8950, 8949, 8950, 8951, 2886, 8950, 8950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=14848 expected_recv=19210 expected_overflow=0 exact_recv_per_rank=[8950, 8950, 8949, 8950, 8951, 2886, 8950, 8950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=14848 expected_recv=20010 expected_overflow=0 exact_recv_per_rank=[8950, 8950, 8949, 8950, 8951, 2886, 8950, 8950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=14848 expected_recv=11400 actual_valid_recv=11648 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=14848 expected_recv=13944 expected_overflow=0 exact_recv_per_rank=[8950, 8950, 8949, 8950, 8951, 2886, 8950, 8950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=14848 expected_recv=19210 actual_valid_recv=19456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=11648 slack_slots=18048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=14848 expected_recv=13944 actual_valid_recv=14208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=14848 expected_recv=20010 actual_valid_recv=20096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=14848 expected_recv=20011 expected_overflow=0 exact_recv_per_rank=[8950, 8950, 8949, 8950, 8951, 2886, 8950, 8950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=14848 expected_recv=20012 expected_overflow=0 exact_recv_per_rank=[8950, 8950, 8949, 8950, 8951, 2886, 8950, 8950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=19456 slack_slots=10240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=14208 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=20096 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=14848 expected_recv=20011 actual_valid_recv=20096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=14848 expected_recv=9395 expected_overflow=0 exact_recv_per_rank=[8950, 8950, 8949, 8950, 8951, 2886, 8950, 8950] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=14848 expected_recv=20012 actual_valid_recv=20352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=14848 expected_recv=7059 expected_overflow=0 exact_recv_per_rank=[9995, 8046, 7383, 13161, 229, 12674, 13160, 888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=20096 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=14848 expected_recv=9395 actual_valid_recv=9600 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=20352 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=14848 expected_recv=7059 actual_valid_recv=7168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=9600 slack_slots=20096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=14848 expected_recv=15539 expected_overflow=0 exact_recv_per_rank=[9995, 8046, 7383, 13161, 229, 12674, 13160, 888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=7168 slack_slots=22528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=14848 expected_recv=15539 actual_valid_recv=15744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=14848 expected_recv=21749 expected_overflow=0 exact_recv_per_rank=[9995, 8046, 7383, 13161, 229, 12674, 13160, 888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=14848 expected_recv=20943 expected_overflow=0 exact_recv_per_rank=[9995, 8046, 7383, 13161, 229, 12674, 13160, 888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=14848 expected_recv=21747 expected_overflow=0 exact_recv_per_rank=[9995, 8046, 7383, 13161, 229, 12674, 13160, 888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=15744 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=14848 expected_recv=16634 expected_overflow=0 exact_recv_per_rank=[9995, 8046, 7383, 13161, 229, 12674, 13160, 888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=14848 expected_recv=21749 actual_valid_recv=21888 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=14848 expected_recv=20943 actual_valid_recv=21120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=14848 expected_recv=21747 actual_valid_recv=22016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=14848 expected_recv=8817 expected_overflow=0 exact_recv_per_rank=[9995, 8046, 7383, 13161, 229, 12674, 13160, 888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=21888 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=14848 expected_recv=16634 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=21120 slack_slots=8576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=22016 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=14848 expected_recv=18584 expected_overflow=0 exact_recv_per_rank=[9995, 8046, 7383, 13161, 229, 12674, 13160, 888] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=14848 expected_recv=12200 expected_overflow=0 exact_recv_per_rank=[11250, 1947, 10694, 11251, 11250, 1671, 10938, 6535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=14848 expected_recv=8817 actual_valid_recv=8960 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=14848 expected_recv=18584 actual_valid_recv=18816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=8960 slack_slots=20736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=14848 expected_recv=12200 actual_valid_recv=12544 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=18816 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=12544 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=14848 expected_recv=19147 expected_overflow=0 exact_recv_per_rank=[11250, 1947, 10694, 11251, 11250, 1671, 10938, 6535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=14848 expected_recv=19147 actual_valid_recv=19328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=14848 expected_recv=20025 expected_overflow=0 exact_recv_per_rank=[11250, 1947, 10694, 11251, 11250, 1671, 10938, 6535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=14848 expected_recv=9219 expected_overflow=0 exact_recv_per_rank=[11250, 1947, 10694, 11251, 11250, 1671, 10938, 6535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=14848 expected_recv=19712 expected_overflow=0 exact_recv_per_rank=[11250, 1947, 10694, 11251, 11250, 1671, 10938, 6535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=14848 expected_recv=10721 expected_overflow=0 exact_recv_per_rank=[11250, 1947, 10694, 11251, 11250, 1671, 10938, 6535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=19328 slack_slots=10368 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=14848 expected_recv=20025 actual_valid_recv=20096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=14848 expected_recv=9219 actual_valid_recv=9344 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=14848 expected_recv=19712 actual_valid_recv=19968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=14848 expected_recv=10721 actual_valid_recv=10880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=20096 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=14848 expected_recv=20024 expected_overflow=0 exact_recv_per_rank=[11250, 1947, 10694, 11251, 11250, 1671, 10938, 6535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=9344 slack_slots=20352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=14848 expected_recv=20024 expected_overflow=0 exact_recv_per_rank=[11250, 1947, 10694, 11251, 11250, 1671, 10938, 6535] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=19968 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=14848 expected_recv=16860 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8019, 8273, 8274, 8273, 7873, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=10880 slack_slots=18816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=14848 expected_recv=20024 actual_valid_recv=20096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=14848 expected_recv=20024 actual_valid_recv=20096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=14848 expected_recv=16860 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=20096 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=20096 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=14848 expected_recv=16604 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8019, 8273, 8274, 8273, 7873, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=14848 expected_recv=16604 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=14848 expected_recv=16858 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8019, 8273, 8274, 8273, 7873, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=14848 expected_recv=16858 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8019, 8273, 8274, 8273, 7873, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=14848 expected_recv=16458 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8019, 8273, 8274, 8273, 7873, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=14848 expected_recv=16858 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=14848 expected_recv=16858 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=14848 expected_recv=16860 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8019, 8273, 8274, 8273, 7873, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=14848 expected_recv=16458 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=14848 expected_recv=14956 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8019, 8273, 8274, 8273, 7873, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=14848 expected_recv=16860 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=14848 expected_recv=15618 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8019, 8273, 8274, 8273, 7873, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[653, 4852, 8995, 2847, 12768, 5980, 13800, 15641] avg=8192.0000 max=15641 max_rank=7 min=653 min_rank=0 imbalance_ratio=1.909302 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=14848 expected_recv=14956 actual_valid_recv=15104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=14848 expected_recv=15618 actual_valid_recv=15872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=15104 slack_slots=14592 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=15872 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[653, 4852, 8995, 2847, 12768, 5980, 13800, 15641] avg=8192.0000 max=15641 max_rank=7 min=653 min_rank=0 imbalance_ratio=1.909302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[7300, 8625, 8995, 7651, 8995, 5980, 8996, 8994] avg=8192.0000 max=8996 max_rank=6 min=5980 min_rank=5 imbalance_ratio=1.098145 max_load_reduction=6645 max_load_reduction_pct=0.424845 moved_to_replicas=32600 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[653, 4852, 8995, 2847, 12768, 5980, 13800, 15641] avg=8192.0000 max=15641 max_rank=7 min=653 min_rank=0 imbalance_ratio=1.909302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[653, 4852, 8995, 2847, 12768, 5980, 13800, 15641] avg=8192.0000 max=15641 max_rank=7 min=653 min_rank=0 imbalance_ratio=1.909302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[653, 4852, 8995, 2847, 12768, 5980, 13800, 15641] avg=8192.0000 max=15641 max_rank=7 min=653 min_rank=0 imbalance_ratio=1.909302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[7300, 8625, 8995, 7651, 8995, 5980, 8996, 8994] avg=8192.0000 max=8996 max_rank=6 min=5980 min_rank=5 imbalance_ratio=1.098145 max_load_reduction=6645 max_load_reduction_pct=0.424845 moved_to_replicas=32600 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[653, 4852, 8995, 2847, 12768, 5980, 13800, 15641] avg=8192.0000 max=15641 max_rank=7 min=653 min_rank=0 imbalance_ratio=1.909302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[653, 4852, 8995, 2847, 12768, 5980, 13800, 15641] avg=8192.0000 max=15641 max_rank=7 min=653 min_rank=0 imbalance_ratio=1.909302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[653, 4852, 8995, 2847, 12768, 5980, 13800, 15641] avg=8192.0000 max=15641 max_rank=7 min=653 min_rank=0 imbalance_ratio=1.909302 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[7300, 8625, 8995, 7651, 8995, 5980, 8996, 8994] avg=8192.0000 max=8996 max_rank=6 min=5980 min_rank=5 imbalance_ratio=1.098145 max_load_reduction=6645 max_load_reduction_pct=0.424845 moved_to_replicas=32600 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[7300, 8625, 8995, 7651, 8995, 5980, 8996, 8994] avg=8192.0000 max=8996 max_rank=6 min=5980 min_rank=5 imbalance_ratio=1.098145 max_load_reduction=6645 max_load_reduction_pct=0.424845 moved_to_replicas=32600 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[15309, 11381, 2000, 30144, 2014, 1637, 781, 2270] avg=8192.0000 max=30144 max_rank=3 min=781 min_rank=6 imbalance_ratio=3.679688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[7300, 8625, 8995, 7651, 8995, 5980, 8996, 8994] avg=8192.0000 max=8996 max_rank=6 min=5980 min_rank=5 imbalance_ratio=1.098145 max_load_reduction=6645 max_load_reduction_pct=0.424845 moved_to_replicas=32600 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[7300, 8625, 8995, 7651, 8995, 5980, 8996, 8994] avg=8192.0000 max=8996 max_rank=6 min=5980 min_rank=5 imbalance_ratio=1.098145 max_load_reduction=6645 max_load_reduction_pct=0.424845 moved_to_replicas=32600 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[7300, 8625, 8995, 7651, 8995, 5980, 8996, 8994] avg=8192.0000 max=8996 max_rank=6 min=5980 min_rank=5 imbalance_ratio=1.098145 max_load_reduction=6645 max_load_reduction_pct=0.424845 moved_to_replicas=32600 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[7300, 8625, 8995, 7651, 8995, 5980, 8996, 8994] avg=8192.0000 max=8996 max_rank=6 min=5980 min_rank=5 imbalance_ratio=1.098145 max_load_reduction=6645 max_load_reduction_pct=0.424845 moved_to_replicas=32600 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[15309, 11381, 2000, 30144, 2014, 1637, 781, 2270] avg=8192.0000 max=30144 max_rank=3 min=781 min_rank=6 imbalance_ratio=3.679688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[8879, 8878, 7927, 8879, 8444, 8878, 8878, 4773] avg=8192.0000 max=8879 max_rank=0 min=4773 min_rank=7 imbalance_ratio=1.083862 max_load_reduction=21265 max_load_reduction_pct=0.705447 moved_to_replicas=60777 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[15309, 11381, 2000, 30144, 2014, 1637, 781, 2270] avg=8192.0000 max=30144 max_rank=3 min=781 min_rank=6 imbalance_ratio=3.679688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[15309, 11381, 2000, 30144, 2014, 1637, 781, 2270] avg=8192.0000 max=30144 max_rank=3 min=781 min_rank=6 imbalance_ratio=3.679688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[15309, 11381, 2000, 30144, 2014, 1637, 781, 2270] avg=8192.0000 max=30144 max_rank=3 min=781 min_rank=6 imbalance_ratio=3.679688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[8879, 8878, 7927, 8879, 8444, 8878, 8878, 4773] avg=8192.0000 max=8879 max_rank=0 min=4773 min_rank=7 imbalance_ratio=1.083862 max_load_reduction=21265 max_load_reduction_pct=0.705447 moved_to_replicas=60777 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[15309, 11381, 2000, 30144, 2014, 1637, 781, 2270] avg=8192.0000 max=30144 max_rank=3 min=781 min_rank=6 imbalance_ratio=3.679688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[15309, 11381, 2000, 30144, 2014, 1637, 781, 2270] avg=8192.0000 max=30144 max_rank=3 min=781 min_rank=6 imbalance_ratio=3.679688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[15309, 11381, 2000, 30144, 2014, 1637, 781, 2270] avg=8192.0000 max=30144 max_rank=3 min=781 min_rank=6 imbalance_ratio=3.679688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[2260, 2050, 1400, 1780, 11556, 1458, 15708, 29324] avg=8192.0000 max=29324 max_rank=7 min=1400 min_rank=2 imbalance_ratio=3.579590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[8879, 8878, 7927, 8879, 8444, 8878, 8878, 4773] avg=8192.0000 max=8879 max_rank=0 min=4773 min_rank=7 imbalance_ratio=1.083862 max_load_reduction=21265 max_load_reduction_pct=0.705447 moved_to_replicas=60777 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[8879, 8878, 7927, 8879, 8444, 8878, 8878, 4773] avg=8192.0000 max=8879 max_rank=0 min=4773 min_rank=7 imbalance_ratio=1.083862 max_load_reduction=21265 max_load_reduction_pct=0.705447 moved_to_replicas=60777 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[8879, 8878, 7927, 8879, 8444, 8878, 8878, 4773] avg=8192.0000 max=8879 max_rank=0 min=4773 min_rank=7 imbalance_ratio=1.083862 max_load_reduction=21265 max_load_reduction_pct=0.705447 moved_to_replicas=60777 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[8879, 8878, 7927, 8879, 8444, 8878, 8878, 4773] avg=8192.0000 max=8879 max_rank=0 min=4773 min_rank=7 imbalance_ratio=1.083862 max_load_reduction=21265 max_load_reduction_pct=0.705447 moved_to_replicas=60777 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[8879, 8878, 7927, 8879, 8444, 8878, 8878, 4773] avg=8192.0000 max=8879 max_rank=0 min=4773 min_rank=7 imbalance_ratio=1.083862 max_load_reduction=21265 max_load_reduction_pct=0.705447 moved_to_replicas=60777 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[4937, 8879, 8879, 7325, 8879, 8879, 8879, 8879] avg=8192.0000 max=8879 max_rank=1 min=4937 min_rank=0 imbalance_ratio=1.083862 max_load_reduction=20445 max_load_reduction_pct=0.697210 moved_to_replicas=54607 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[8879, 8878, 7927, 8879, 8444, 8878, 8878, 4773] avg=8192.0000 max=8879 max_rank=0 min=4773 min_rank=7 imbalance_ratio=1.083862 max_load_reduction=21265 max_load_reduction_pct=0.705447 moved_to_replicas=60777 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[2260, 2050, 1400, 1780, 11556, 1458, 15708, 29324] avg=8192.0000 max=29324 max_rank=7 min=1400 min_rank=2 imbalance_ratio=3.579590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[2260, 2050, 1400, 1780, 11556, 1458, 15708, 29324] avg=8192.0000 max=29324 max_rank=7 min=1400 min_rank=2 imbalance_ratio=3.579590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[2260, 2050, 1400, 1780, 11556, 1458, 15708, 29324] avg=8192.0000 max=29324 max_rank=7 min=1400 min_rank=2 imbalance_ratio=3.579590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[2260, 2050, 1400, 1780, 11556, 1458, 15708, 29324] avg=8192.0000 max=29324 max_rank=7 min=1400 min_rank=2 imbalance_ratio=3.579590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[2260, 2050, 1400, 1780, 11556, 1458, 15708, 29324] avg=8192.0000 max=29324 max_rank=7 min=1400 min_rank=2 imbalance_ratio=3.579590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[4937, 8879, 8879, 7325, 8879, 8879, 8879, 8879] avg=8192.0000 max=8879 max_rank=1 min=4937 min_rank=0 imbalance_ratio=1.083862 max_load_reduction=20445 max_load_reduction_pct=0.697210 moved_to_replicas=54607 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[7783, 757, 8211, 762, 5932, 16078, 24534, 1479] avg=8192.0000 max=24534 max_rank=6 min=757 min_rank=1 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[2260, 2050, 1400, 1780, 11556, 1458, 15708, 29324] avg=8192.0000 max=29324 max_rank=7 min=1400 min_rank=2 imbalance_ratio=3.579590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[2260, 2050, 1400, 1780, 11556, 1458, 15708, 29324] avg=8192.0000 max=29324 max_rank=7 min=1400 min_rank=2 imbalance_ratio=3.579590 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[4937, 8879, 8879, 7325, 8879, 8879, 8879, 8879] avg=8192.0000 max=8879 max_rank=1 min=4937 min_rank=0 imbalance_ratio=1.083862 max_load_reduction=20445 max_load_reduction_pct=0.697210 moved_to_replicas=54607 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[4937, 8879, 8879, 7325, 8879, 8879, 8879, 8879] avg=8192.0000 max=8879 max_rank=1 min=4937 min_rank=0 imbalance_ratio=1.083862 max_load_reduction=20445 max_load_reduction_pct=0.697210 moved_to_replicas=54607 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[4937, 8879, 8879, 7325, 8879, 8879, 8879, 8879] avg=8192.0000 max=8879 max_rank=1 min=4937 min_rank=0 imbalance_ratio=1.083862 max_load_reduction=20445 max_load_reduction_pct=0.697210 moved_to_replicas=54607 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[4937, 8879, 8879, 7325, 8879, 8879, 8879, 8879] avg=8192.0000 max=8879 max_rank=1 min=4937 min_rank=0 imbalance_ratio=1.083862 max_load_reduction=20445 max_load_reduction_pct=0.697210 moved_to_replicas=54607 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[7783, 8765, 8211, 8522, 5957, 8766, 8766, 8766] avg=8192.0000 max=8766 max_rank=5 min=5957 min_rank=4 imbalance_ratio=1.070068 max_load_reduction=15768 max_load_reduction_pct=0.642700 moved_to_replicas=51972 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[4937, 8879, 8879, 7325, 8879, 8879, 8879, 8879] avg=8192.0000 max=8879 max_rank=1 min=4937 min_rank=0 imbalance_ratio=1.083862 max_load_reduction=20445 max_load_reduction_pct=0.697210 moved_to_replicas=54607 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[7783, 757, 8211, 762, 5932, 16078, 24534, 1479] avg=8192.0000 max=24534 max_rank=6 min=757 min_rank=1 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[4937, 8879, 8879, 7325, 8879, 8879, 8879, 8879] avg=8192.0000 max=8879 max_rank=1 min=4937 min_rank=0 imbalance_ratio=1.083862 max_load_reduction=20445 max_load_reduction_pct=0.697210 moved_to_replicas=54607 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[7783, 757, 8211, 762, 5932, 16078, 24534, 1479] avg=8192.0000 max=24534 max_rank=6 min=757 min_rank=1 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[7783, 757, 8211, 762, 5932, 16078, 24534, 1479] avg=8192.0000 max=24534 max_rank=6 min=757 min_rank=1 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[7783, 8765, 8211, 8522, 5957, 8766, 8766, 8766] avg=8192.0000 max=8766 max_rank=5 min=5957 min_rank=4 imbalance_ratio=1.070068 max_load_reduction=15768 max_load_reduction_pct=0.642700 moved_to_replicas=51972 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[7783, 757, 8211, 762, 5932, 16078, 24534, 1479] avg=8192.0000 max=24534 max_rank=6 min=757 min_rank=1 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[7783, 757, 8211, 762, 5932, 16078, 24534, 1479] avg=8192.0000 max=24534 max_rank=6 min=757 min_rank=1 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[1029, 8090, 8400, 24900, 185, 218, 22519, 195] avg=8192.0000 max=24900 max_rank=3 min=185 min_rank=4 imbalance_ratio=3.039551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[7783, 757, 8211, 762, 5932, 16078, 24534, 1479] avg=8192.0000 max=24534 max_rank=6 min=757 min_rank=1 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[7783, 8765, 8211, 8522, 5957, 8766, 8766, 8766] avg=8192.0000 max=8766 max_rank=5 min=5957 min_rank=4 imbalance_ratio=1.070068 max_load_reduction=15768 max_load_reduction_pct=0.642700 moved_to_replicas=51972 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[7783, 757, 8211, 762, 5932, 16078, 24534, 1479] avg=8192.0000 max=24534 max_rank=6 min=757 min_rank=1 imbalance_ratio=2.994873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[7783, 8765, 8211, 8522, 5957, 8766, 8766, 8766] avg=8192.0000 max=8766 max_rank=5 min=5957 min_rank=4 imbalance_ratio=1.070068 max_load_reduction=15768 max_load_reduction_pct=0.642700 moved_to_replicas=51972 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[7783, 8765, 8211, 8522, 5957, 8766, 8766, 8766] avg=8192.0000 max=8766 max_rank=5 min=5957 min_rank=4 imbalance_ratio=1.070068 max_load_reduction=15768 max_load_reduction_pct=0.642700 moved_to_replicas=51972 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[7783, 8765, 8211, 8522, 5957, 8766, 8766, 8766] avg=8192.0000 max=8766 max_rank=5 min=5957 min_rank=4 imbalance_ratio=1.070068 max_load_reduction=15768 max_load_reduction_pct=0.642700 moved_to_replicas=51972 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[6720, 8090, 8400, 8523, 8522, 8523, 8523, 8235] avg=8192.0000 max=8523 max_rank=3 min=6720 min_rank=0 imbalance_ratio=1.040405 max_load_reduction=16377 max_load_reduction_pct=0.657711 moved_to_replicas=65235 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[1029, 8090, 8400, 24900, 185, 218, 22519, 195] avg=8192.0000 max=24900 max_rank=3 min=185 min_rank=4 imbalance_ratio=3.039551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[7783, 8765, 8211, 8522, 5957, 8766, 8766, 8766] avg=8192.0000 max=8766 max_rank=5 min=5957 min_rank=4 imbalance_ratio=1.070068 max_load_reduction=15768 max_load_reduction_pct=0.642700 moved_to_replicas=51972 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[7783, 8765, 8211, 8522, 5957, 8766, 8766, 8766] avg=8192.0000 max=8766 max_rank=5 min=5957 min_rank=4 imbalance_ratio=1.070068 max_load_reduction=15768 max_load_reduction_pct=0.642700 moved_to_replicas=51972 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[1029, 8090, 8400, 24900, 185, 218, 22519, 195] avg=8192.0000 max=24900 max_rank=3 min=185 min_rank=4 imbalance_ratio=3.039551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[1029, 8090, 8400, 24900, 185, 218, 22519, 195] avg=8192.0000 max=24900 max_rank=3 min=185 min_rank=4 imbalance_ratio=3.039551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[6720, 8090, 8400, 8523, 8522, 8523, 8523, 8235] avg=8192.0000 max=8523 max_rank=3 min=6720 min_rank=0 imbalance_ratio=1.040405 max_load_reduction=16377 max_load_reduction_pct=0.657711 moved_to_replicas=65235 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[1029, 8090, 8400, 24900, 185, 218, 22519, 195] avg=8192.0000 max=24900 max_rank=3 min=185 min_rank=4 imbalance_ratio=3.039551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[1029, 8090, 8400, 24900, 185, 218, 22519, 195] avg=8192.0000 max=24900 max_rank=3 min=185 min_rank=4 imbalance_ratio=3.039551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[437, 32089, 500, 1409, 30, 30855, 216, 0] avg=8192.0000 max=32089 max_rank=1 min=0 min_rank=7 imbalance_ratio=3.917114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[1029, 8090, 8400, 24900, 185, 218, 22519, 195] avg=8192.0000 max=24900 max_rank=3 min=185 min_rank=4 imbalance_ratio=3.039551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[6720, 8090, 8400, 8523, 8522, 8523, 8523, 8235] avg=8192.0000 max=8523 max_rank=3 min=6720 min_rank=0 imbalance_ratio=1.040405 max_load_reduction=16377 max_load_reduction_pct=0.657711 moved_to_replicas=65235 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[1029, 8090, 8400, 24900, 185, 218, 22519, 195] avg=8192.0000 max=24900 max_rank=3 min=185 min_rank=4 imbalance_ratio=3.039551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[6720, 8090, 8400, 8523, 8522, 8523, 8523, 8235] avg=8192.0000 max=8523 max_rank=3 min=6720 min_rank=0 imbalance_ratio=1.040405 max_load_reduction=16377 max_load_reduction_pct=0.657711 moved_to_replicas=65235 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[6720, 8090, 8400, 8523, 8522, 8523, 8523, 8235] avg=8192.0000 max=8523 max_rank=3 min=6720 min_rank=0 imbalance_ratio=1.040405 max_load_reduction=16377 max_load_reduction_pct=0.657711 moved_to_replicas=65235 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[8595, 8596, 8595, 7416, 8594, 8595, 6550, 8595] avg=8192.0000 max=8596 max_rank=1 min=6550 min_rank=6 imbalance_ratio=1.049316 max_load_reduction=23493 max_load_reduction_pct=0.732120 moved_to_replicas=93738 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[6720, 8090, 8400, 8523, 8522, 8523, 8523, 8235] avg=8192.0000 max=8523 max_rank=3 min=6720 min_rank=0 imbalance_ratio=1.040405 max_load_reduction=16377 max_load_reduction_pct=0.657711 moved_to_replicas=65235 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[437, 32089, 500, 1409, 30, 30855, 216, 0] avg=8192.0000 max=32089 max_rank=1 min=0 min_rank=7 imbalance_ratio=3.917114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[6720, 8090, 8400, 8523, 8522, 8523, 8523, 8235] avg=8192.0000 max=8523 max_rank=3 min=6720 min_rank=0 imbalance_ratio=1.040405 max_load_reduction=16377 max_load_reduction_pct=0.657711 moved_to_replicas=65235 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[6720, 8090, 8400, 8523, 8522, 8523, 8523, 8235] avg=8192.0000 max=8523 max_rank=3 min=6720 min_rank=0 imbalance_ratio=1.040405 max_load_reduction=16377 max_load_reduction_pct=0.657711 moved_to_replicas=65235 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[437, 32089, 500, 1409, 30, 30855, 216, 0] avg=8192.0000 max=32089 max_rank=1 min=0 min_rank=7 imbalance_ratio=3.917114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[437, 32089, 500, 1409, 30, 30855, 216, 0] avg=8192.0000 max=32089 max_rank=1 min=0 min_rank=7 imbalance_ratio=3.917114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[8595, 8596, 8595, 7416, 8594, 8595, 6550, 8595] avg=8192.0000 max=8596 max_rank=1 min=6550 min_rank=6 imbalance_ratio=1.049316 max_load_reduction=23493 max_load_reduction_pct=0.732120 moved_to_replicas=93738 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[437, 32089, 500, 1409, 30, 30855, 216, 0] avg=8192.0000 max=32089 max_rank=1 min=0 min_rank=7 imbalance_ratio=3.917114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[37, 7779, 29010, 318, 1222, 558, 545, 26067] avg=8192.0000 max=29010 max_rank=2 min=37 min_rank=0 imbalance_ratio=3.541260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[437, 32089, 500, 1409, 30, 30855, 216, 0] avg=8192.0000 max=32089 max_rank=1 min=0 min_rank=7 imbalance_ratio=3.917114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[8595, 8596, 8595, 7416, 8594, 8595, 6550, 8595] avg=8192.0000 max=8596 max_rank=1 min=6550 min_rank=6 imbalance_ratio=1.049316 max_load_reduction=23493 max_load_reduction_pct=0.732120 moved_to_replicas=93738 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[437, 32089, 500, 1409, 30, 30855, 216, 0] avg=8192.0000 max=32089 max_rank=1 min=0 min_rank=7 imbalance_ratio=3.917114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[8595, 8596, 8595, 7416, 8594, 8595, 6550, 8595] avg=8192.0000 max=8596 max_rank=1 min=6550 min_rank=6 imbalance_ratio=1.049316 max_load_reduction=23493 max_load_reduction_pct=0.732120 moved_to_replicas=93738 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[437, 32089, 500, 1409, 30, 30855, 216, 0] avg=8192.0000 max=32089 max_rank=1 min=0 min_rank=7 imbalance_ratio=3.917114 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[8595, 8596, 8595, 7416, 8594, 8595, 6550, 8595] avg=8192.0000 max=8596 max_rank=1 min=6550 min_rank=6 imbalance_ratio=1.049316 max_load_reduction=23493 max_load_reduction_pct=0.732120 moved_to_replicas=93738 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[9284, 7779, 9282, 9283, 9282, 9282, 2061, 9283] avg=8192.0000 max=9284 max_rank=0 min=2061 min_rank=6 imbalance_ratio=1.133301 max_load_reduction=19726 max_load_reduction_pct=0.679972 moved_to_replicas=73201 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[8595, 8596, 8595, 7416, 8594, 8595, 6550, 8595] avg=8192.0000 max=8596 max_rank=1 min=6550 min_rank=6 imbalance_ratio=1.049316 max_load_reduction=23493 max_load_reduction_pct=0.732120 moved_to_replicas=93738 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[37, 7779, 29010, 318, 1222, 558, 545, 26067] avg=8192.0000 max=29010 max_rank=2 min=37 min_rank=0 imbalance_ratio=3.541260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[8595, 8596, 8595, 7416, 8594, 8595, 6550, 8595] avg=8192.0000 max=8596 max_rank=1 min=6550 min_rank=6 imbalance_ratio=1.049316 max_load_reduction=23493 max_load_reduction_pct=0.732120 moved_to_replicas=93738 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[8595, 8596, 8595, 7416, 8594, 8595, 6550, 8595] avg=8192.0000 max=8596 max_rank=1 min=6550 min_rank=6 imbalance_ratio=1.049316 max_load_reduction=23493 max_load_reduction_pct=0.732120 moved_to_replicas=93738 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[37, 7779, 29010, 318, 1222, 558, 545, 26067] avg=8192.0000 max=29010 max_rank=2 min=37 min_rank=0 imbalance_ratio=3.541260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[37, 7779, 29010, 318, 1222, 558, 545, 26067] avg=8192.0000 max=29010 max_rank=2 min=37 min_rank=0 imbalance_ratio=3.541260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[9284, 7779, 9282, 9283, 9282, 9282, 2061, 9283] avg=8192.0000 max=9284 max_rank=0 min=2061 min_rank=6 imbalance_ratio=1.133301 max_load_reduction=19726 max_load_reduction_pct=0.679972 moved_to_replicas=73201 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[229, 53, 20576, 6239, 30473, 3513, 472, 3981] avg=8192.0000 max=30473 max_rank=4 min=53 min_rank=1 imbalance_ratio=3.719849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[37, 7779, 29010, 318, 1222, 558, 545, 26067] avg=8192.0000 max=29010 max_rank=2 min=37 min_rank=0 imbalance_ratio=3.541260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[37, 7779, 29010, 318, 1222, 558, 545, 26067] avg=8192.0000 max=29010 max_rank=2 min=37 min_rank=0 imbalance_ratio=3.541260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[9284, 7779, 9282, 9283, 9282, 9282, 2061, 9283] avg=8192.0000 max=9284 max_rank=0 min=2061 min_rank=6 imbalance_ratio=1.133301 max_load_reduction=19726 max_load_reduction_pct=0.679972 moved_to_replicas=73201 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[37, 7779, 29010, 318, 1222, 558, 545, 26067] avg=8192.0000 max=29010 max_rank=2 min=37 min_rank=0 imbalance_ratio=3.541260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[9284, 7779, 9282, 9283, 9282, 9282, 2061, 9283] avg=8192.0000 max=9284 max_rank=0 min=2061 min_rank=6 imbalance_ratio=1.133301 max_load_reduction=19726 max_load_reduction_pct=0.679972 moved_to_replicas=73201 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[8578, 8580, 8580, 8572, 8579, 8578, 5490, 8579] avg=8192.0000 max=8580 max_rank=1 min=5490 min_rank=6 imbalance_ratio=1.047363 max_load_reduction=21893 max_load_reduction_pct=0.718439 moved_to_replicas=70657 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[37, 7779, 29010, 318, 1222, 558, 545, 26067] avg=8192.0000 max=29010 max_rank=2 min=37 min_rank=0 imbalance_ratio=3.541260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[229, 53, 20576, 6239, 30473, 3513, 472, 3981] avg=8192.0000 max=30473 max_rank=4 min=53 min_rank=1 imbalance_ratio=3.719849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[9284, 7779, 9282, 9283, 9282, 9282, 2061, 9283] avg=8192.0000 max=9284 max_rank=0 min=2061 min_rank=6 imbalance_ratio=1.133301 max_load_reduction=19726 max_load_reduction_pct=0.679972 moved_to_replicas=73201 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[9284, 7779, 9282, 9283, 9282, 9282, 2061, 9283] avg=8192.0000 max=9284 max_rank=0 min=2061 min_rank=6 imbalance_ratio=1.133301 max_load_reduction=19726 max_load_reduction_pct=0.679972 moved_to_replicas=73201 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[9284, 7779, 9282, 9283, 9282, 9282, 2061, 9283] avg=8192.0000 max=9284 max_rank=0 min=2061 min_rank=6 imbalance_ratio=1.133301 max_load_reduction=19726 max_load_reduction_pct=0.679972 moved_to_replicas=73201 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[229, 53, 20576, 6239, 30473, 3513, 472, 3981] avg=8192.0000 max=30473 max_rank=4 min=53 min_rank=1 imbalance_ratio=3.719849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[9284, 7779, 9282, 9283, 9282, 9282, 2061, 9283] avg=8192.0000 max=9284 max_rank=0 min=2061 min_rank=6 imbalance_ratio=1.133301 max_load_reduction=19726 max_load_reduction_pct=0.679972 moved_to_replicas=73201 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[229, 53, 20576, 6239, 30473, 3513, 472, 3981] avg=8192.0000 max=30473 max_rank=4 min=53 min_rank=1 imbalance_ratio=3.719849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[8578, 8580, 8580, 8572, 8579, 8578, 5490, 8579] avg=8192.0000 max=8580 max_rank=1 min=5490 min_rank=6 imbalance_ratio=1.047363 max_load_reduction=21893 max_load_reduction_pct=0.718439 moved_to_replicas=70657 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[3098, 11508, 0, 967, 27633, 619, 79, 21632] avg=8192.0000 max=27633 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.373169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[229, 53, 20576, 6239, 30473, 3513, 472, 3981] avg=8192.0000 max=30473 max_rank=4 min=53 min_rank=1 imbalance_ratio=3.719849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[8578, 8580, 8580, 8572, 8579, 8578, 5490, 8579] avg=8192.0000 max=8580 max_rank=1 min=5490 min_rank=6 imbalance_ratio=1.047363 max_load_reduction=21893 max_load_reduction_pct=0.718439 moved_to_replicas=70657 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[229, 53, 20576, 6239, 30473, 3513, 472, 3981] avg=8192.0000 max=30473 max_rank=4 min=53 min_rank=1 imbalance_ratio=3.719849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[8578, 8580, 8580, 8572, 8579, 8578, 5490, 8579] avg=8192.0000 max=8580 max_rank=1 min=5490 min_rank=6 imbalance_ratio=1.047363 max_load_reduction=21893 max_load_reduction_pct=0.718439 moved_to_replicas=70657 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[229, 53, 20576, 6239, 30473, 3513, 472, 3981] avg=8192.0000 max=30473 max_rank=4 min=53 min_rank=1 imbalance_ratio=3.719849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[5368, 9238, 9238, 4742, 9237, 9238, 9237, 9238] avg=8192.0000 max=9238 max_rank=1 min=4742 min_rank=3 imbalance_ratio=1.127686 max_load_reduction=18395 max_load_reduction_pct=0.665690 moved_to_replicas=68470 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[229, 53, 20576, 6239, 30473, 3513, 472, 3981] avg=8192.0000 max=30473 max_rank=4 min=53 min_rank=1 imbalance_ratio=3.719849 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[3098, 11508, 0, 967, 27633, 619, 79, 21632] avg=8192.0000 max=27633 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.373169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[8578, 8580, 8580, 8572, 8579, 8578, 5490, 8579] avg=8192.0000 max=8580 max_rank=1 min=5490 min_rank=6 imbalance_ratio=1.047363 max_load_reduction=21893 max_load_reduction_pct=0.718439 moved_to_replicas=70657 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[8578, 8580, 8580, 8572, 8579, 8578, 5490, 8579] avg=8192.0000 max=8580 max_rank=1 min=5490 min_rank=6 imbalance_ratio=1.047363 max_load_reduction=21893 max_load_reduction_pct=0.718439 moved_to_replicas=70657 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[8578, 8580, 8580, 8572, 8579, 8578, 5490, 8579] avg=8192.0000 max=8580 max_rank=1 min=5490 min_rank=6 imbalance_ratio=1.047363 max_load_reduction=21893 max_load_reduction_pct=0.718439 moved_to_replicas=70657 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[3098, 11508, 0, 967, 27633, 619, 79, 21632] avg=8192.0000 max=27633 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.373169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[8578, 8580, 8580, 8572, 8579, 8578, 5490, 8579] avg=8192.0000 max=8580 max_rank=1 min=5490 min_rank=6 imbalance_ratio=1.047363 max_load_reduction=21893 max_load_reduction_pct=0.718439 moved_to_replicas=70657 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[3098, 11508, 0, 967, 27633, 619, 79, 21632] avg=8192.0000 max=27633 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.373169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[5368, 9238, 9238, 4742, 9237, 9238, 9237, 9238] avg=8192.0000 max=9238 max_rank=1 min=4742 min_rank=3 imbalance_ratio=1.127686 max_load_reduction=18395 max_load_reduction_pct=0.665690 moved_to_replicas=68470 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[279, 3461, 763, 16238, 18681, 61, 369, 25684] avg=8192.0000 max=25684 max_rank=7 min=61 min_rank=5 imbalance_ratio=3.135254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[3098, 11508, 0, 967, 27633, 619, 79, 21632] avg=8192.0000 max=27633 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.373169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[5368, 9238, 9238, 4742, 9237, 9238, 9237, 9238] avg=8192.0000 max=9238 max_rank=1 min=4742 min_rank=3 imbalance_ratio=1.127686 max_load_reduction=18395 max_load_reduction_pct=0.665690 moved_to_replicas=68470 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[5368, 9238, 9238, 4742, 9237, 9238, 9237, 9238] avg=8192.0000 max=9238 max_rank=1 min=4742 min_rank=3 imbalance_ratio=1.127686 max_load_reduction=18395 max_load_reduction_pct=0.665690 moved_to_replicas=68470 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[3098, 11508, 0, 967, 27633, 619, 79, 21632] avg=8192.0000 max=27633 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.373169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[6973, 3461, 7475, 9526, 9525, 9526, 9525, 9525] avg=8192.0000 max=9526 max_rank=3 min=3461 min_rank=1 imbalance_ratio=1.162842 max_load_reduction=16158 max_load_reduction_pct=0.629108 moved_to_replicas=61656 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[3098, 11508, 0, 967, 27633, 619, 79, 21632] avg=8192.0000 max=27633 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.373169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[3098, 11508, 0, 967, 27633, 619, 79, 21632] avg=8192.0000 max=27633 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.373169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[279, 3461, 763, 16238, 18681, 61, 369, 25684] avg=8192.0000 max=25684 max_rank=7 min=61 min_rank=5 imbalance_ratio=3.135254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[5368, 9238, 9238, 4742, 9237, 9238, 9237, 9238] avg=8192.0000 max=9238 max_rank=1 min=4742 min_rank=3 imbalance_ratio=1.127686 max_load_reduction=18395 max_load_reduction_pct=0.665690 moved_to_replicas=68470 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[5368, 9238, 9238, 4742, 9237, 9238, 9237, 9238] avg=8192.0000 max=9238 max_rank=1 min=4742 min_rank=3 imbalance_ratio=1.127686 max_load_reduction=18395 max_load_reduction_pct=0.665690 moved_to_replicas=68470 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[279, 3461, 763, 16238, 18681, 61, 369, 25684] avg=8192.0000 max=25684 max_rank=7 min=61 min_rank=5 imbalance_ratio=3.135254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[5368, 9238, 9238, 4742, 9237, 9238, 9237, 9238] avg=8192.0000 max=9238 max_rank=1 min=4742 min_rank=3 imbalance_ratio=1.127686 max_load_reduction=18395 max_load_reduction_pct=0.665690 moved_to_replicas=68470 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[279, 3461, 763, 16238, 18681, 61, 369, 25684] avg=8192.0000 max=25684 max_rank=7 min=61 min_rank=5 imbalance_ratio=3.135254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[5368, 9238, 9238, 4742, 9237, 9238, 9237, 9238] avg=8192.0000 max=9238 max_rank=1 min=4742 min_rank=3 imbalance_ratio=1.127686 max_load_reduction=18395 max_load_reduction_pct=0.665690 moved_to_replicas=68470 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[3, 92, 1268, 15161, 20073, 26182, 411, 2346] avg=8192.0000 max=26182 max_rank=5 min=3 min_rank=0 imbalance_ratio=3.196045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[6973, 3461, 7475, 9526, 9525, 9526, 9525, 9525] avg=8192.0000 max=9526 max_rank=3 min=3461 min_rank=1 imbalance_ratio=1.162842 max_load_reduction=16158 max_load_reduction_pct=0.629108 moved_to_replicas=61656 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[279, 3461, 763, 16238, 18681, 61, 369, 25684] avg=8192.0000 max=25684 max_rank=7 min=61 min_rank=5 imbalance_ratio=3.135254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[6973, 3461, 7475, 9526, 9525, 9526, 9525, 9525] avg=8192.0000 max=9526 max_rank=3 min=3461 min_rank=1 imbalance_ratio=1.162842 max_load_reduction=16158 max_load_reduction_pct=0.629108 moved_to_replicas=61656 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[6973, 3461, 7475, 9526, 9525, 9526, 9525, 9525] avg=8192.0000 max=9526 max_rank=3 min=3461 min_rank=1 imbalance_ratio=1.162842 max_load_reduction=16158 max_load_reduction_pct=0.629108 moved_to_replicas=61656 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[8799, 8680, 4156, 8798, 8798, 8798, 8798, 8709] avg=8192.0000 max=8799 max_rank=0 min=4156 min_rank=2 imbalance_ratio=1.074097 max_load_reduction=17383 max_load_reduction_pct=0.663929 moved_to_replicas=72935 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[279, 3461, 763, 16238, 18681, 61, 369, 25684] avg=8192.0000 max=25684 max_rank=7 min=61 min_rank=5 imbalance_ratio=3.135254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[279, 3461, 763, 16238, 18681, 61, 369, 25684] avg=8192.0000 max=25684 max_rank=7 min=61 min_rank=5 imbalance_ratio=3.135254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[279, 3461, 763, 16238, 18681, 61, 369, 25684] avg=8192.0000 max=25684 max_rank=7 min=61 min_rank=5 imbalance_ratio=3.135254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[3, 92, 1268, 15161, 20073, 26182, 411, 2346] avg=8192.0000 max=26182 max_rank=5 min=3 min_rank=0 imbalance_ratio=3.196045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[6973, 3461, 7475, 9526, 9525, 9526, 9525, 9525] avg=8192.0000 max=9526 max_rank=3 min=3461 min_rank=1 imbalance_ratio=1.162842 max_load_reduction=16158 max_load_reduction_pct=0.629108 moved_to_replicas=61656 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[3, 92, 1268, 15161, 20073, 26182, 411, 2346] avg=8192.0000 max=26182 max_rank=5 min=3 min_rank=0 imbalance_ratio=3.196045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[6973, 3461, 7475, 9526, 9525, 9526, 9525, 9525] avg=8192.0000 max=9526 max_rank=3 min=3461 min_rank=1 imbalance_ratio=1.162842 max_load_reduction=16158 max_load_reduction_pct=0.629108 moved_to_replicas=61656 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[6973, 3461, 7475, 9526, 9525, 9526, 9525, 9525] avg=8192.0000 max=9526 max_rank=3 min=3461 min_rank=1 imbalance_ratio=1.162842 max_load_reduction=16158 max_load_reduction_pct=0.629108 moved_to_replicas=61656 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[3, 92, 1268, 15161, 20073, 26182, 411, 2346] avg=8192.0000 max=26182 max_rank=5 min=3 min_rank=0 imbalance_ratio=3.196045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[13776, 6328, 31488, 423, 3575, 761, 8911, 274] avg=8192.0000 max=31488 max_rank=2 min=274 min_rank=7 imbalance_ratio=3.843750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[6973, 3461, 7475, 9526, 9525, 9526, 9525, 9525] avg=8192.0000 max=9526 max_rank=3 min=3461 min_rank=1 imbalance_ratio=1.162842 max_load_reduction=16158 max_load_reduction_pct=0.629108 moved_to_replicas=61656 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[8799, 8680, 4156, 8798, 8798, 8798, 8798, 8709] avg=8192.0000 max=8799 max_rank=0 min=4156 min_rank=2 imbalance_ratio=1.074097 max_load_reduction=17383 max_load_reduction_pct=0.663929 moved_to_replicas=72935 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[8799, 8680, 4156, 8798, 8798, 8798, 8798, 8709] avg=8192.0000 max=8799 max_rank=0 min=4156 min_rank=2 imbalance_ratio=1.074097 max_load_reduction=17383 max_load_reduction_pct=0.663929 moved_to_replicas=72935 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[8799, 8680, 4156, 8798, 8798, 8798, 8798, 8709] avg=8192.0000 max=8799 max_rank=0 min=4156 min_rank=2 imbalance_ratio=1.074097 max_load_reduction=17383 max_load_reduction_pct=0.663929 moved_to_replicas=72935 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[3, 92, 1268, 15161, 20073, 26182, 411, 2346] avg=8192.0000 max=26182 max_rank=5 min=3 min_rank=0 imbalance_ratio=3.196045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[8904, 6335, 8904, 8904, 8447, 6234, 8904, 8904] avg=8192.0000 max=8904 max_rank=0 min=6234 min_rank=5 imbalance_ratio=1.086914 max_load_reduction=22584 max_load_reduction_pct=0.717226 moved_to_replicas=60644 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[3, 92, 1268, 15161, 20073, 26182, 411, 2346] avg=8192.0000 max=26182 max_rank=5 min=3 min_rank=0 imbalance_ratio=3.196045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[3, 92, 1268, 15161, 20073, 26182, 411, 2346] avg=8192.0000 max=26182 max_rank=5 min=3 min_rank=0 imbalance_ratio=3.196045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[13776, 6328, 31488, 423, 3575, 761, 8911, 274] avg=8192.0000 max=31488 max_rank=2 min=274 min_rank=7 imbalance_ratio=3.843750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[3, 92, 1268, 15161, 20073, 26182, 411, 2346] avg=8192.0000 max=26182 max_rank=5 min=3 min_rank=0 imbalance_ratio=3.196045 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[8799, 8680, 4156, 8798, 8798, 8798, 8798, 8709] avg=8192.0000 max=8799 max_rank=0 min=4156 min_rank=2 imbalance_ratio=1.074097 max_load_reduction=17383 max_load_reduction_pct=0.663929 moved_to_replicas=72935 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[13776, 6328, 31488, 423, 3575, 761, 8911, 274] avg=8192.0000 max=31488 max_rank=2 min=274 min_rank=7 imbalance_ratio=3.843750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[13776, 6328, 31488, 423, 3575, 761, 8911, 274] avg=8192.0000 max=31488 max_rank=2 min=274 min_rank=7 imbalance_ratio=3.843750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[8799, 8680, 4156, 8798, 8798, 8798, 8798, 8709] avg=8192.0000 max=8799 max_rank=0 min=4156 min_rank=2 imbalance_ratio=1.074097 max_load_reduction=17383 max_load_reduction_pct=0.663929 moved_to_replicas=72935 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[1188, 27114, 149, 3488, 38, 1186, 28363, 4010] avg=8192.0000 max=28363 max_rank=6 min=38 min_rank=4 imbalance_ratio=3.462280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[8799, 8680, 4156, 8798, 8798, 8798, 8798, 8709] avg=8192.0000 max=8799 max_rank=0 min=4156 min_rank=2 imbalance_ratio=1.074097 max_load_reduction=17383 max_load_reduction_pct=0.663929 moved_to_replicas=72935 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[8904, 6335, 8904, 8904, 8447, 6234, 8904, 8904] avg=8192.0000 max=8904 max_rank=0 min=6234 min_rank=5 imbalance_ratio=1.086914 max_load_reduction=22584 max_load_reduction_pct=0.717226 moved_to_replicas=60644 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[8799, 8680, 4156, 8798, 8798, 8798, 8798, 8709] avg=8192.0000 max=8799 max_rank=0 min=4156 min_rank=2 imbalance_ratio=1.074097 max_load_reduction=17383 max_load_reduction_pct=0.663929 moved_to_replicas=72935 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[8904, 6335, 8904, 8904, 8447, 6234, 8904, 8904] avg=8192.0000 max=8904 max_rank=0 min=6234 min_rank=5 imbalance_ratio=1.086914 max_load_reduction=22584 max_load_reduction_pct=0.717226 moved_to_replicas=60644 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[8904, 6335, 8904, 8904, 8447, 6234, 8904, 8904] avg=8192.0000 max=8904 max_rank=0 min=6234 min_rank=5 imbalance_ratio=1.086914 max_load_reduction=22584 max_load_reduction_pct=0.717226 moved_to_replicas=60644 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[13776, 6328, 31488, 423, 3575, 761, 8911, 274] avg=8192.0000 max=31488 max_rank=2 min=274 min_rank=7 imbalance_ratio=3.843750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[8950, 8950, 8949, 8950, 8951, 2886, 8950, 8950] avg=8192.0000 max=8951 max_rank=4 min=2886 min_rank=5 imbalance_ratio=1.092651 max_load_reduction=19412 max_load_reduction_pct=0.684413 moved_to_replicas=61680 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[13776, 6328, 31488, 423, 3575, 761, 8911, 274] avg=8192.0000 max=31488 max_rank=2 min=274 min_rank=7 imbalance_ratio=3.843750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[1188, 27114, 149, 3488, 38, 1186, 28363, 4010] avg=8192.0000 max=28363 max_rank=6 min=38 min_rank=4 imbalance_ratio=3.462280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[13776, 6328, 31488, 423, 3575, 761, 8911, 274] avg=8192.0000 max=31488 max_rank=2 min=274 min_rank=7 imbalance_ratio=3.843750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[13776, 6328, 31488, 423, 3575, 761, 8911, 274] avg=8192.0000 max=31488 max_rank=2 min=274 min_rank=7 imbalance_ratio=3.843750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[1188, 27114, 149, 3488, 38, 1186, 28363, 4010] avg=8192.0000 max=28363 max_rank=6 min=38 min_rank=4 imbalance_ratio=3.462280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[8904, 6335, 8904, 8904, 8447, 6234, 8904, 8904] avg=8192.0000 max=8904 max_rank=0 min=6234 min_rank=5 imbalance_ratio=1.086914 max_load_reduction=22584 max_load_reduction_pct=0.717226 moved_to_replicas=60644 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[1188, 27114, 149, 3488, 38, 1186, 28363, 4010] avg=8192.0000 max=28363 max_rank=6 min=38 min_rank=4 imbalance_ratio=3.462280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[9995, 8046, 7383, 26259, 229, 12674, 62, 888] avg=8192.0000 max=26259 max_rank=3 min=62 min_rank=6 imbalance_ratio=3.205444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[8904, 6335, 8904, 8904, 8447, 6234, 8904, 8904] avg=8192.0000 max=8904 max_rank=0 min=6234 min_rank=5 imbalance_ratio=1.086914 max_load_reduction=22584 max_load_reduction_pct=0.717226 moved_to_replicas=60644 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[8950, 8950, 8949, 8950, 8951, 2886, 8950, 8950] avg=8192.0000 max=8951 max_rank=4 min=2886 min_rank=5 imbalance_ratio=1.092651 max_load_reduction=19412 max_load_reduction_pct=0.684413 moved_to_replicas=61680 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[8904, 6335, 8904, 8904, 8447, 6234, 8904, 8904] avg=8192.0000 max=8904 max_rank=0 min=6234 min_rank=5 imbalance_ratio=1.086914 max_load_reduction=22584 max_load_reduction_pct=0.717226 moved_to_replicas=60644 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[8950, 8950, 8949, 8950, 8951, 2886, 8950, 8950] avg=8192.0000 max=8951 max_rank=4 min=2886 min_rank=5 imbalance_ratio=1.092651 max_load_reduction=19412 max_load_reduction_pct=0.684413 moved_to_replicas=61680 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[8904, 6335, 8904, 8904, 8447, 6234, 8904, 8904] avg=8192.0000 max=8904 max_rank=0 min=6234 min_rank=5 imbalance_ratio=1.086914 max_load_reduction=22584 max_load_reduction_pct=0.717226 moved_to_replicas=60644 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[1188, 27114, 149, 3488, 38, 1186, 28363, 4010] avg=8192.0000 max=28363 max_rank=6 min=38 min_rank=4 imbalance_ratio=3.462280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[8950, 8950, 8949, 8950, 8951, 2886, 8950, 8950] avg=8192.0000 max=8951 max_rank=4 min=2886 min_rank=5 imbalance_ratio=1.092651 max_load_reduction=19412 max_load_reduction_pct=0.684413 moved_to_replicas=61680 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[9995, 8046, 7383, 13161, 229, 12674, 13160, 888] avg=8192.0000 max=13161 max_rank=3 min=229 min_rank=4 imbalance_ratio=1.606567 max_load_reduction=13098 max_load_reduction_pct=0.498800 moved_to_replicas=42428 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[1188, 27114, 149, 3488, 38, 1186, 28363, 4010] avg=8192.0000 max=28363 max_rank=6 min=38 min_rank=4 imbalance_ratio=3.462280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[9995, 8046, 7383, 26259, 229, 12674, 62, 888] avg=8192.0000 max=26259 max_rank=3 min=62 min_rank=6 imbalance_ratio=3.205444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[9995, 8046, 7383, 26259, 229, 12674, 62, 888] avg=8192.0000 max=26259 max_rank=3 min=62 min_rank=6 imbalance_ratio=3.205444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[1188, 27114, 149, 3488, 38, 1186, 28363, 4010] avg=8192.0000 max=28363 max_rank=6 min=38 min_rank=4 imbalance_ratio=3.462280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[8950, 8950, 8949, 8950, 8951, 2886, 8950, 8950] avg=8192.0000 max=8951 max_rank=4 min=2886 min_rank=5 imbalance_ratio=1.092651 max_load_reduction=19412 max_load_reduction_pct=0.684413 moved_to_replicas=61680 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[1188, 27114, 149, 3488, 38, 1186, 28363, 4010] avg=8192.0000 max=28363 max_rank=6 min=38 min_rank=4 imbalance_ratio=3.462280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[9995, 8046, 7383, 26259, 229, 12674, 62, 888] avg=8192.0000 max=26259 max_rank=3 min=62 min_rank=6 imbalance_ratio=3.205444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[11250, 1947, 10694, 22123, 12801, 120, 66, 6535] avg=8192.0000 max=22123 max_rank=3 min=66 min_rank=6 imbalance_ratio=2.700562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[9995, 8046, 7383, 13161, 229, 12674, 13160, 888] avg=8192.0000 max=13161 max_rank=3 min=229 min_rank=4 imbalance_ratio=1.606567 max_load_reduction=13098 max_load_reduction_pct=0.498800 moved_to_replicas=42428 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[8950, 8950, 8949, 8950, 8951, 2886, 8950, 8950] avg=8192.0000 max=8951 max_rank=4 min=2886 min_rank=5 imbalance_ratio=1.092651 max_load_reduction=19412 max_load_reduction_pct=0.684413 moved_to_replicas=61680 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[9995, 8046, 7383, 13161, 229, 12674, 13160, 888] avg=8192.0000 max=13161 max_rank=3 min=229 min_rank=4 imbalance_ratio=1.606567 max_load_reduction=13098 max_load_reduction_pct=0.498800 moved_to_replicas=42428 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[8950, 8950, 8949, 8950, 8951, 2886, 8950, 8950] avg=8192.0000 max=8951 max_rank=4 min=2886 min_rank=5 imbalance_ratio=1.092651 max_load_reduction=19412 max_load_reduction_pct=0.684413 moved_to_replicas=61680 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[8950, 8950, 8949, 8950, 8951, 2886, 8950, 8950] avg=8192.0000 max=8951 max_rank=4 min=2886 min_rank=5 imbalance_ratio=1.092651 max_load_reduction=19412 max_load_reduction_pct=0.684413 moved_to_replicas=61680 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[9995, 8046, 7383, 13161, 229, 12674, 13160, 888] avg=8192.0000 max=13161 max_rank=3 min=229 min_rank=4 imbalance_ratio=1.606567 max_load_reduction=13098 max_load_reduction_pct=0.498800 moved_to_replicas=42428 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[9995, 8046, 7383, 26259, 229, 12674, 62, 888] avg=8192.0000 max=26259 max_rank=3 min=62 min_rank=6 imbalance_ratio=3.205444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[11250, 1947, 10694, 11251, 11250, 1671, 10938, 6535] avg=8192.0000 max=11251 max_rank=3 min=1671 min_rank=5 imbalance_ratio=1.373413 max_load_reduction=10872 max_load_reduction_pct=0.491434 moved_to_replicas=38753 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[11250, 1947, 10694, 22123, 12801, 120, 66, 6535] avg=8192.0000 max=22123 max_rank=3 min=66 min_rank=6 imbalance_ratio=2.700562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[11250, 1947, 10694, 22123, 12801, 120, 66, 6535] avg=8192.0000 max=22123 max_rank=3 min=66 min_rank=6 imbalance_ratio=2.700562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[9995, 8046, 7383, 26259, 229, 12674, 62, 888] avg=8192.0000 max=26259 max_rank=3 min=62 min_rank=6 imbalance_ratio=3.205444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[9995, 8046, 7383, 26259, 229, 12674, 62, 888] avg=8192.0000 max=26259 max_rank=3 min=62 min_rank=6 imbalance_ratio=3.205444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[9995, 8046, 7383, 13161, 229, 12674, 13160, 888] avg=8192.0000 max=13161 max_rank=3 min=229 min_rank=4 imbalance_ratio=1.606567 max_load_reduction=13098 max_load_reduction_pct=0.498800 moved_to_replicas=42428 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[9995, 8046, 7383, 26259, 229, 12674, 62, 888] avg=8192.0000 max=26259 max_rank=3 min=62 min_rank=6 imbalance_ratio=3.205444 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[1419, 89, 377, 25393, 106, 32269, 5029, 854] avg=8192.0000 max=32269 max_rank=5 min=89 min_rank=1 imbalance_ratio=3.939087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[11250, 1947, 10694, 11251, 11250, 1671, 10938, 6535] avg=8192.0000 max=11251 max_rank=3 min=1671 min_rank=5 imbalance_ratio=1.373413 max_load_reduction=10872 max_load_reduction_pct=0.491434 moved_to_replicas=38753 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[11250, 1947, 10694, 11251, 11250, 1671, 10938, 6535] avg=8192.0000 max=11251 max_rank=3 min=1671 min_rank=5 imbalance_ratio=1.373413 max_load_reduction=10872 max_load_reduction_pct=0.491434 moved_to_replicas=38753 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[11250, 1947, 10694, 22123, 12801, 120, 66, 6535] avg=8192.0000 max=22123 max_rank=3 min=66 min_rank=6 imbalance_ratio=2.700562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[9995, 8046, 7383, 13161, 229, 12674, 13160, 888] avg=8192.0000 max=13161 max_rank=3 min=229 min_rank=4 imbalance_ratio=1.606567 max_load_reduction=13098 max_load_reduction_pct=0.498800 moved_to_replicas=42428 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[9995, 8046, 7383, 13161, 229, 12674, 13160, 888] avg=8192.0000 max=13161 max_rank=3 min=229 min_rank=4 imbalance_ratio=1.606567 max_load_reduction=13098 max_load_reduction_pct=0.498800 moved_to_replicas=42428 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[9995, 8046, 7383, 13161, 229, 12674, 13160, 888] avg=8192.0000 max=13161 max_rank=3 min=229 min_rank=4 imbalance_ratio=1.606567 max_load_reduction=13098 max_load_reduction_pct=0.498800 moved_to_replicas=42428 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[8274, 8275, 8019, 8273, 8274, 8273, 7873, 8275] avg=8192.0000 max=8275 max_rank=1 min=7873 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=23994 max_load_reduction_pct=0.743562 moved_to_replicas=82265 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[11250, 1947, 10694, 22123, 12801, 120, 66, 6535] avg=8192.0000 max=22123 max_rank=3 min=66 min_rank=6 imbalance_ratio=2.700562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[11250, 1947, 10694, 11251, 11250, 1671, 10938, 6535] avg=8192.0000 max=11251 max_rank=3 min=1671 min_rank=5 imbalance_ratio=1.373413 max_load_reduction=10872 max_load_reduction_pct=0.491434 moved_to_replicas=38753 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[1419, 89, 377, 25393, 106, 32269, 5029, 854] avg=8192.0000 max=32269 max_rank=5 min=89 min_rank=1 imbalance_ratio=3.939087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[1419, 89, 377, 25393, 106, 32269, 5029, 854] avg=8192.0000 max=32269 max_rank=5 min=89 min_rank=1 imbalance_ratio=3.939087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[11250, 1947, 10694, 22123, 12801, 120, 66, 6535] avg=8192.0000 max=22123 max_rank=3 min=66 min_rank=6 imbalance_ratio=2.700562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[11250, 1947, 10694, 22123, 12801, 120, 66, 6535] avg=8192.0000 max=22123 max_rank=3 min=66 min_rank=6 imbalance_ratio=2.700562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[11250, 1947, 10694, 11251, 11250, 1671, 10938, 6535] avg=8192.0000 max=11251 max_rank=3 min=1671 min_rank=5 imbalance_ratio=1.373413 max_load_reduction=10872 max_load_reduction_pct=0.491434 moved_to_replicas=38753 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[11250, 1947, 10694, 22123, 12801, 120, 66, 6535] avg=8192.0000 max=22123 max_rank=3 min=66 min_rank=6 imbalance_ratio=2.700562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[1419, 89, 377, 25393, 106, 32269, 5029, 854] avg=8192.0000 max=32269 max_rank=5 min=89 min_rank=1 imbalance_ratio=3.939087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[8274, 8275, 8019, 8273, 8274, 8273, 7873, 8275] avg=8192.0000 max=8275 max_rank=1 min=7873 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=23994 max_load_reduction_pct=0.743562 moved_to_replicas=82265 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[8274, 8275, 8019, 8273, 8274, 8273, 7873, 8275] avg=8192.0000 max=8275 max_rank=1 min=7873 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=23994 max_load_reduction_pct=0.743562 moved_to_replicas=82265 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[11250, 1947, 10694, 11251, 11250, 1671, 10938, 6535] avg=8192.0000 max=11251 max_rank=3 min=1671 min_rank=5 imbalance_ratio=1.373413 max_load_reduction=10872 max_load_reduction_pct=0.491434 moved_to_replicas=38753 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[11250, 1947, 10694, 11251, 11250, 1671, 10938, 6535] avg=8192.0000 max=11251 max_rank=3 min=1671 min_rank=5 imbalance_ratio=1.373413 max_load_reduction=10872 max_load_reduction_pct=0.491434 moved_to_replicas=38753 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[11250, 1947, 10694, 11251, 11250, 1671, 10938, 6535] avg=8192.0000 max=11251 max_rank=3 min=1671 min_rank=5 imbalance_ratio=1.373413 max_load_reduction=10872 max_load_reduction_pct=0.491434 moved_to_replicas=38753 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[1419, 89, 377, 25393, 106, 32269, 5029, 854] avg=8192.0000 max=32269 max_rank=5 min=89 min_rank=1 imbalance_ratio=3.939087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[8274, 8275, 8019, 8273, 8274, 8273, 7873, 8275] avg=8192.0000 max=8275 max_rank=1 min=7873 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=23994 max_load_reduction_pct=0.743562 moved_to_replicas=82265 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[1419, 89, 377, 25393, 106, 32269, 5029, 854] avg=8192.0000 max=32269 max_rank=5 min=89 min_rank=1 imbalance_ratio=3.939087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[1419, 89, 377, 25393, 106, 32269, 5029, 854] avg=8192.0000 max=32269 max_rank=5 min=89 min_rank=1 imbalance_ratio=3.939087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[8274, 8275, 8019, 8273, 8274, 8273, 7873, 8275] avg=8192.0000 max=8275 max_rank=1 min=7873 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=23994 max_load_reduction_pct=0.743562 moved_to_replicas=82265 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[1419, 89, 377, 25393, 106, 32269, 5029, 854] avg=8192.0000 max=32269 max_rank=5 min=89 min_rank=1 imbalance_ratio=3.939087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[8274, 8275, 8019, 8273, 8274, 8273, 7873, 8275] avg=8192.0000 max=8275 max_rank=1 min=7873 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=23994 max_load_reduction_pct=0.743562 moved_to_replicas=82265 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[8274, 8275, 8019, 8273, 8274, 8273, 7873, 8275] avg=8192.0000 max=8275 max_rank=1 min=7873 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=23994 max_load_reduction_pct=0.743562 moved_to_replicas=82265 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[8274, 8275, 8019, 8273, 8274, 8273, 7873, 8275] avg=8192.0000 max=8275 max_rank=1 min=7873 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=23994 max_load_reduction_pct=0.743562 moved_to_replicas=82265 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=32 total_expected_recv=250965 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=220928 peak_demand=10790 required_capacity=1.317139 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=32 total_expected_recv=263222 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=208896 peak_demand=12674 required_capacity=1.547119 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=32 total_expected_recv=265072 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=206848 peak_demand=10791 required_capacity=1.317261 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=32 total_expected_recv=268931 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=202624 peak_demand=13161 required_capacity=1.606567 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=32 total_expected_recv=271569 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=199680 peak_demand=13160 required_capacity=1.606445 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=32 total_expected_recv=252710 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=219264 peak_demand=11061 required_capacity=1.350220 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=32 total_expected_recv=275797 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=196096 peak_demand=11250 required_capacity=1.373291 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=32 total_expected_recv=248886 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=222464 peak_demand=11250 required_capacity=1.373291 + [2026-08-30 05:09:12] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 1225.9 | throughput per GPU (TFLOP/s/GPU): 271.9 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 9.679976E+00 | loss scale: 1.0 | grad norm: 42.146 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 13312 | min safe capacity: 1.6250 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=14848 expected_recv=17168 expected_overflow=0 exact_recv_per_rank=[8581, 8583, 8229, 7137, 8583, 7307, 8534, 8582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=14848 expected_recv=16777 expected_overflow=0 exact_recv_per_rank=[8581, 8583, 8229, 7137, 8583, 7307, 8534, 8582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=14848 expected_recv=15451 expected_overflow=0 exact_recv_per_rank=[8581, 8583, 8229, 7137, 8583, 7307, 8534, 8582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=14848 expected_recv=14643 expected_overflow=0 exact_recv_per_rank=[8581, 8583, 8229, 7137, 8583, 7307, 8534, 8582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=14848 expected_recv=17167 expected_overflow=0 exact_recv_per_rank=[8581, 8583, 8229, 7137, 8583, 7307, 8534, 8582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=14848 expected_recv=15534 expected_overflow=0 exact_recv_per_rank=[8581, 8583, 8229, 7137, 8583, 7307, 8534, 8582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=14848 expected_recv=17165 expected_overflow=0 exact_recv_per_rank=[8581, 8583, 8229, 7137, 8583, 7307, 8534, 8582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=14848 expected_recv=17168 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=14848 expected_recv=16777 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=14848 expected_recv=15451 actual_valid_recv=15616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=14848 expected_recv=17167 expected_overflow=0 exact_recv_per_rank=[8581, 8583, 8229, 7137, 8583, 7307, 8534, 8582] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=14848 expected_recv=14643 actual_valid_recv=14848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=14848 expected_recv=17167 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17408 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=15616 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=14848 expected_recv=15534 actual_valid_recv=15616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=14848 expected_recv=17165 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=14848 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17408 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=14848 expected_recv=17167 actual_valid_recv=17408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=15616 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17536 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17408 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=14848 expected_recv=7732 expected_overflow=0 exact_recv_per_rank=[9078, 9079, 4005, 9078, 9078, 9078, 7062, 9078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=14848 expected_recv=18171 expected_overflow=0 exact_recv_per_rank=[9078, 9079, 4005, 9078, 9078, 9078, 7062, 9078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=14848 expected_recv=18173 expected_overflow=0 exact_recv_per_rank=[9078, 9079, 4005, 9078, 9078, 9078, 7062, 9078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=14848 expected_recv=18171 expected_overflow=0 exact_recv_per_rank=[9078, 9079, 4005, 9078, 9078, 9078, 7062, 9078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=14848 expected_recv=7732 actual_valid_recv=8064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=14848 expected_recv=18170 expected_overflow=0 exact_recv_per_rank=[9078, 9079, 4005, 9078, 9078, 9078, 7062, 9078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=14848 expected_recv=18171 actual_valid_recv=18432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=8064 slack_slots=21632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=14848 expected_recv=18173 actual_valid_recv=18432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=14848 expected_recv=14313 expected_overflow=0 exact_recv_per_rank=[9078, 9079, 4005, 9078, 9078, 9078, 7062, 9078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=14848 expected_recv=18171 expected_overflow=0 exact_recv_per_rank=[9078, 9079, 4005, 9078, 9078, 9078, 7062, 9078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=14848 expected_recv=18171 actual_valid_recv=18432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=14848 expected_recv=18171 expected_overflow=0 exact_recv_per_rank=[9078, 9079, 4005, 9078, 9078, 9078, 7062, 9078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=14848 expected_recv=18170 actual_valid_recv=18304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=18432 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=18432 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=14848 expected_recv=14313 actual_valid_recv=14720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=14848 expected_recv=18171 actual_valid_recv=18304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=18432 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=18304 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=14848 expected_recv=18171 actual_valid_recv=18304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=14720 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=18304 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=18304 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=14848 expected_recv=15662 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7904, 8274, 8274, 7988, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=14848 expected_recv=15662 actual_valid_recv=16000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=14848 expected_recv=16262 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7904, 8274, 8274, 7988, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=14848 expected_recv=16408 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7904, 8274, 8274, 7988, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16000 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7904, 8274, 8274, 7988, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7904, 8274, 8274, 7988, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=14848 expected_recv=16262 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=14848 expected_recv=16408 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7904, 8274, 8274, 7988, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7904, 8274, 8274, 7988, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7904, 8274, 8274, 7988, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=14848 expected_recv=17778 expected_overflow=0 exact_recv_per_rank=[8890, 8890, 8890, 4144, 8890, 8890, 8052, 8890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=14848 expected_recv=12962 expected_overflow=0 exact_recv_per_rank=[8890, 8890, 8890, 4144, 8890, 8890, 8052, 8890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=14848 expected_recv=17778 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=14848 expected_recv=17778 expected_overflow=0 exact_recv_per_rank=[8890, 8890, 8890, 4144, 8890, 8890, 8052, 8890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=17920 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=14848 expected_recv=12962 actual_valid_recv=13312 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=14848 expected_recv=17778 actual_valid_recv=18176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=14848 expected_recv=13031 expected_overflow=0 exact_recv_per_rank=[8890, 8890, 8890, 4144, 8890, 8890, 8052, 8890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=14848 expected_recv=17778 expected_overflow=0 exact_recv_per_rank=[8890, 8890, 8890, 4144, 8890, 8890, 8052, 8890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=13312 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=14848 expected_recv=16188 expected_overflow=0 exact_recv_per_rank=[8890, 8890, 8890, 4144, 8890, 8890, 8052, 8890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=18176 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=14848 expected_recv=13031 actual_valid_recv=13440 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=14848 expected_recv=17778 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=14848 expected_recv=17779 expected_overflow=0 exact_recv_per_rank=[8890, 8890, 8890, 4144, 8890, 8890, 8052, 8890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=14848 expected_recv=17778 expected_overflow=0 exact_recv_per_rank=[8890, 8890, 8890, 4144, 8890, 8890, 8052, 8890] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=14848 expected_recv=16188 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=13440 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=17920 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=14848 expected_recv=19608 expected_overflow=0 exact_recv_per_rank=[4005, 9610, 9610, 9611, 9609, 9609, 3872, 9610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=14848 expected_recv=17779 actual_valid_recv=18176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=14848 expected_recv=17778 actual_valid_recv=18048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=14848 expected_recv=19605 expected_overflow=0 exact_recv_per_rank=[4005, 9610, 9610, 9611, 9609, 9609, 3872, 9610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=18176 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=14848 expected_recv=19608 actual_valid_recv=19840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=18048 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=14848 expected_recv=19607 expected_overflow=0 exact_recv_per_rank=[4005, 9610, 9610, 9611, 9609, 9609, 3872, 9610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=14848 expected_recv=19605 actual_valid_recv=19840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=19840 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=14848 expected_recv=19607 actual_valid_recv=19840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=19840 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=19840 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=14848 expected_recv=19608 expected_overflow=0 exact_recv_per_rank=[4005, 9610, 9610, 9611, 9609, 9609, 3872, 9610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=14848 expected_recv=19607 expected_overflow=0 exact_recv_per_rank=[4005, 9610, 9610, 9611, 9609, 9609, 3872, 9610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=14848 expected_recv=6538 expected_overflow=0 exact_recv_per_rank=[4005, 9610, 9610, 9611, 9609, 9609, 3872, 9610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=14848 expected_recv=19608 actual_valid_recv=20096 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=14848 expected_recv=19607 actual_valid_recv=19840 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=14848 expected_recv=19606 expected_overflow=0 exact_recv_per_rank=[4005, 9610, 9610, 9611, 9609, 9609, 3872, 9610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=14848 expected_recv=6893 expected_overflow=0 exact_recv_per_rank=[4005, 9610, 9610, 9611, 9609, 9609, 3872, 9610] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=14848 expected_recv=6538 actual_valid_recv=6912 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=20096 slack_slots=9600 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8274, 8274, 7800, 8274, 8094, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=19840 slack_slots=9856 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=14848 expected_recv=19606 actual_valid_recv=19968 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=6912 slack_slots=22784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=14848 expected_recv=6893 actual_valid_recv=7168 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8274, 8274, 7800, 8274, 8094, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8274, 8274, 7800, 8274, 8094, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=19968 slack_slots=9728 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=7168 slack_slots=22528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8274, 8274, 7800, 8274, 8094, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8274, 8274, 7800, 8274, 8094, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=14848 expected_recv=16215 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8274, 8274, 7800, 8274, 8094, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=14848 expected_recv=15571 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8274, 8274, 7800, 8274, 8094, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=14848 expected_recv=16215 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8274, 8274, 7800, 8274, 8094, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8274, 8274, 7961, 8274, 8274, 7932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=14848 expected_recv=15571 actual_valid_recv=15872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8274, 8274, 7961, 8274, 8274, 7932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=15872 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=14848 expected_recv=16242 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8274, 8274, 7961, 8274, 8274, 7932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=14848 expected_recv=16242 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8274, 8274, 7961, 8274, 8274, 7932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=14848 expected_recv=16207 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8274, 8274, 7961, 8274, 8274, 7932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8274, 8274, 7961, 8274, 8274, 7932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=14848 expected_recv=15883 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8274, 8274, 7961, 8274, 8274, 7932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=14848 expected_recv=16207 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=14848 expected_recv=15883 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8274, 8274, 7961, 8274, 8274, 7932] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=14848 expected_recv=16875 expected_overflow=0 exact_recv_per_rank=[8602, 7592, 8601, 8602, 8603, 7970, 6964, 8602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=14848 expected_recv=16244 expected_overflow=0 exact_recv_per_rank=[8602, 7592, 8601, 8602, 8603, 7970, 6964, 8602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=14848 expected_recv=16875 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=14848 expected_recv=16244 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=17280 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=14848 expected_recv=15707 expected_overflow=0 exact_recv_per_rank=[8602, 7592, 8601, 8602, 8603, 7970, 6964, 8602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=14848 expected_recv=15707 actual_valid_recv=15872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=14848 expected_recv=16875 expected_overflow=0 exact_recv_per_rank=[8602, 7592, 8601, 8602, 8603, 7970, 6964, 8602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=14848 expected_recv=16744 expected_overflow=0 exact_recv_per_rank=[8602, 7592, 8601, 8602, 8603, 7970, 6964, 8602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=14848 expected_recv=14874 expected_overflow=0 exact_recv_per_rank=[8602, 7592, 8601, 8602, 8603, 7970, 6964, 8602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=15872 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=14848 expected_recv=16877 expected_overflow=0 exact_recv_per_rank=[8602, 7592, 8601, 8602, 8603, 7970, 6964, 8602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=14848 expected_recv=16875 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=14848 expected_recv=16744 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=14848 expected_recv=14874 actual_valid_recv=15232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=14848 expected_recv=16877 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=15232 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=14848 expected_recv=16876 expected_overflow=0 exact_recv_per_rank=[8602, 7592, 8601, 8602, 8603, 7970, 6964, 8602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 7934, 8274, 7957, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=14848 expected_recv=15932 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 7934, 8274, 7957, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=14848 expected_recv=16876 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=14848 expected_recv=15932 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 7934, 8274, 7957, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=17280 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=14848 expected_recv=15852 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 7934, 8274, 7957, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 7934, 8274, 7957, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 7934, 8274, 7957, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=14848 expected_recv=15852 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 7934, 8274, 7957, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=14848 expected_recv=17535 expected_overflow=0 exact_recv_per_rank=[8289, 8930, 8931, 8930, 5962, 8930, 8931, 6633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 7934, 8274, 7957, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=14848 expected_recv=17531 expected_overflow=0 exact_recv_per_rank=[8289, 8930, 8931, 8930, 5962, 8930, 8931, 6633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=14848 expected_recv=17535 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=14848 expected_recv=17531 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=17664 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=14848 expected_recv=17532 expected_overflow=0 exact_recv_per_rank=[8289, 8930, 8931, 8930, 5962, 8930, 8931, 6633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=17920 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=14848 expected_recv=17532 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=14848 expected_recv=17532 expected_overflow=0 exact_recv_per_rank=[8289, 8930, 8931, 8930, 5962, 8930, 8931, 6633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=14848 expected_recv=13930 expected_overflow=0 exact_recv_per_rank=[8289, 8930, 8931, 8930, 5962, 8930, 8931, 6633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=17664 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=14848 expected_recv=17532 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=14848 expected_recv=14256 expected_overflow=0 exact_recv_per_rank=[8289, 8930, 8931, 8930, 5962, 8930, 8931, 6633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=14848 expected_recv=17533 expected_overflow=0 exact_recv_per_rank=[8289, 8930, 8931, 8930, 5962, 8930, 8931, 6633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=14848 expected_recv=13930 actual_valid_recv=14208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=17920 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=14848 expected_recv=14256 actual_valid_recv=14592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=14208 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=14848 expected_recv=17533 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 7942, 8275, 8273, 8275, 8274, 8273, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=14848 expected_recv=15223 expected_overflow=0 exact_recv_per_rank=[8289, 8930, 8931, 8930, 5962, 8930, 8931, 6633] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 7942, 8275, 8273, 8275, 8274, 8273, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=14592 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=17664 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=14848 expected_recv=15223 actual_valid_recv=15616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=15616 slack_slots=14080 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=14848 expected_recv=15859 expected_overflow=0 exact_recv_per_rank=[8275, 7942, 8275, 8273, 8275, 8274, 8273, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 7942, 8275, 8273, 8275, 8274, 8273, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=14848 expected_recv=15924 expected_overflow=0 exact_recv_per_rank=[8275, 7942, 8275, 8273, 8275, 8274, 8273, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=14848 expected_recv=15859 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 7942, 8275, 8273, 8275, 8274, 8273, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=14848 expected_recv=15924 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 7942, 8275, 8273, 8275, 8274, 8273, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=14848 expected_recv=9206 expected_overflow=0 exact_recv_per_rank=[8931, 8930, 5990, 6344, 8930, 8551, 8929, 8931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 7942, 8275, 8273, 8275, 8274, 8273, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=14848 expected_recv=18406 expected_overflow=0 exact_recv_per_rank=[8931, 8930, 5990, 6344, 8930, 8551, 8929, 8931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=14848 expected_recv=9206 actual_valid_recv=9472 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=14848 expected_recv=18406 actual_valid_recv=18688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=9472 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=18688 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=14848 expected_recv=18785 expected_overflow=0 exact_recv_per_rank=[8931, 8930, 5990, 6344, 8930, 8551, 8929, 8931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=14848 expected_recv=16201 expected_overflow=0 exact_recv_per_rank=[8931, 8930, 5990, 6344, 8930, 8551, 8929, 8931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=14848 expected_recv=18786 expected_overflow=0 exact_recv_per_rank=[8931, 8930, 5990, 6344, 8930, 8551, 8929, 8931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=14848 expected_recv=18788 expected_overflow=0 exact_recv_per_rank=[8931, 8930, 5990, 6344, 8930, 8551, 8929, 8931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=14848 expected_recv=16201 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=14848 expected_recv=18785 actual_valid_recv=18816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=14848 expected_recv=18786 actual_valid_recv=19072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=14848 expected_recv=18788 actual_valid_recv=18944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=18816 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=14848 expected_recv=12114 expected_overflow=0 exact_recv_per_rank=[8931, 8930, 5990, 6344, 8930, 8551, 8929, 8931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=19072 slack_slots=10624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=14848 expected_recv=15931 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7963, 8274, 8274, 8274, 7930, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7963, 8274, 8274, 8274, 7930, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=14848 expected_recv=18786 expected_overflow=0 exact_recv_per_rank=[8931, 8930, 5990, 6344, 8930, 8551, 8929, 8931] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=18944 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=14848 expected_recv=12114 actual_valid_recv=12416 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=14848 expected_recv=15931 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=14848 expected_recv=18786 actual_valid_recv=18816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=12416 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=18816 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7963, 8274, 8274, 8274, 7930, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7963, 8274, 8274, 8274, 7930, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7963, 8274, 8274, 8274, 7930, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7963, 8274, 8274, 8274, 7930, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=14848 expected_recv=16183 expected_overflow=0 exact_recv_per_rank=[7942, 8274, 8275, 8274, 7948, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=14848 expected_recv=16203 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7963, 8274, 8274, 8274, 7930, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7942, 8274, 8275, 8274, 7948, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=14848 expected_recv=16197 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 7963, 8274, 8274, 8274, 7930, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=14848 expected_recv=16183 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=14848 expected_recv=16203 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=14848 expected_recv=16197 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7942, 8274, 8275, 8274, 7948, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7942, 8274, 8275, 8274, 7948, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7942, 8274, 8275, 8274, 7948, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=14848 expected_recv=15932 expected_overflow=0 exact_recv_per_rank=[7942, 8274, 8275, 8274, 7948, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8275, 8274, 8273, 7931, 8274, 7962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=14848 expected_recv=16205 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8275, 8274, 8273, 7931, 8274, 7962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7942, 8274, 8275, 8274, 7948, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=14848 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[7942, 8274, 8275, 8274, 7948, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=14848 expected_recv=15932 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=14848 expected_recv=16205 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=14848 expected_recv=16217 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8275, 8274, 8273, 7931, 8274, 7962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=14848 expected_recv=15921 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8275, 8274, 8273, 7931, 8274, 7962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8275, 8274, 8273, 7931, 8274, 7962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=14848 expected_recv=15921 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8275, 8274, 8273, 7931, 8274, 7962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7987, 7904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7987, 7904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8275, 8274, 8273, 7931, 8274, 7962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=14848 expected_recv=16204 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8275, 8274, 8273, 7931, 8274, 7962] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=14848 expected_recv=16204 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=14848 expected_recv=16177 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7987, 7904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=14848 expected_recv=16177 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=14848 expected_recv=16177 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7987, 7904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=14848 expected_recv=16177 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7987, 7904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7987, 7904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[301, 295, 2673, 5732, 9988, 7307, 8534, 30706] avg=8192.0000 max=30706 max_rank=7 min=295 min_rank=1 imbalance_ratio=3.748291 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=14848 expected_recv=15977 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7987, 7904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7987, 7904] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[301, 295, 2673, 5732, 9988, 7307, 8534, 30706] avg=8192.0000 max=30706 max_rank=7 min=295 min_rank=1 imbalance_ratio=3.748291 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=14848 expected_recv=15977 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[301, 295, 2673, 5732, 9988, 7307, 8534, 30706] avg=8192.0000 max=30706 max_rank=7 min=295 min_rank=1 imbalance_ratio=3.748291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[8581, 8583, 8229, 7137, 8583, 7307, 8534, 8582] avg=8192.0000 max=8583 max_rank=1 min=7137 min_rank=3 imbalance_ratio=1.047729 max_load_reduction=22123 max_load_reduction_pct=0.720478 moved_to_replicas=47952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[8581, 8583, 8229, 7137, 8583, 7307, 8534, 8582] avg=8192.0000 max=8583 max_rank=1 min=7137 min_rank=3 imbalance_ratio=1.047729 max_load_reduction=22123 max_load_reduction_pct=0.720478 moved_to_replicas=47952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[301, 295, 2673, 5732, 9988, 7307, 8534, 30706] avg=8192.0000 max=30706 max_rank=7 min=295 min_rank=1 imbalance_ratio=3.748291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[301, 295, 2673, 5732, 9988, 7307, 8534, 30706] avg=8192.0000 max=30706 max_rank=7 min=295 min_rank=1 imbalance_ratio=3.748291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[8581, 8583, 8229, 7137, 8583, 7307, 8534, 8582] avg=8192.0000 max=8583 max_rank=1 min=7137 min_rank=3 imbalance_ratio=1.047729 max_load_reduction=22123 max_load_reduction_pct=0.720478 moved_to_replicas=47952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[301, 295, 2673, 5732, 9988, 7307, 8534, 30706] avg=8192.0000 max=30706 max_rank=7 min=295 min_rank=1 imbalance_ratio=3.748291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[301, 295, 2673, 5732, 9988, 7307, 8534, 30706] avg=8192.0000 max=30706 max_rank=7 min=295 min_rank=1 imbalance_ratio=3.748291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[24773, 2, 9, 1, 9078, 180, 265, 31228] avg=8192.0000 max=31228 max_rank=7 min=1 min_rank=3 imbalance_ratio=3.812012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[301, 295, 2673, 5732, 9988, 7307, 8534, 30706] avg=8192.0000 max=30706 max_rank=7 min=295 min_rank=1 imbalance_ratio=3.748291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[8581, 8583, 8229, 7137, 8583, 7307, 8534, 8582] avg=8192.0000 max=8583 max_rank=1 min=7137 min_rank=3 imbalance_ratio=1.047729 max_load_reduction=22123 max_load_reduction_pct=0.720478 moved_to_replicas=47952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[24773, 2, 9, 1, 9078, 180, 265, 31228] avg=8192.0000 max=31228 max_rank=7 min=1 min_rank=3 imbalance_ratio=3.812012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[8581, 8583, 8229, 7137, 8583, 7307, 8534, 8582] avg=8192.0000 max=8583 max_rank=1 min=7137 min_rank=3 imbalance_ratio=1.047729 max_load_reduction=22123 max_load_reduction_pct=0.720478 moved_to_replicas=47952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[8581, 8583, 8229, 7137, 8583, 7307, 8534, 8582] avg=8192.0000 max=8583 max_rank=1 min=7137 min_rank=3 imbalance_ratio=1.047729 max_load_reduction=22123 max_load_reduction_pct=0.720478 moved_to_replicas=47952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[24773, 2, 9, 1, 9078, 180, 265, 31228] avg=8192.0000 max=31228 max_rank=7 min=1 min_rank=3 imbalance_ratio=3.812012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[9078, 9079, 4005, 9078, 9078, 9078, 7062, 9078] avg=8192.0000 max=9079 max_rank=1 min=4005 min_rank=2 imbalance_ratio=1.108276 max_load_reduction=22149 max_load_reduction_pct=0.709267 moved_to_replicas=75187 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[8581, 8583, 8229, 7137, 8583, 7307, 8534, 8582] avg=8192.0000 max=8583 max_rank=1 min=7137 min_rank=3 imbalance_ratio=1.047729 max_load_reduction=22123 max_load_reduction_pct=0.720478 moved_to_replicas=47952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[8581, 8583, 8229, 7137, 8583, 7307, 8534, 8582] avg=8192.0000 max=8583 max_rank=1 min=7137 min_rank=3 imbalance_ratio=1.047729 max_load_reduction=22123 max_load_reduction_pct=0.720478 moved_to_replicas=47952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[9078, 9079, 4005, 9078, 9078, 9078, 7062, 9078] avg=8192.0000 max=9079 max_rank=1 min=4005 min_rank=2 imbalance_ratio=1.108276 max_load_reduction=22149 max_load_reduction_pct=0.709267 moved_to_replicas=75187 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[24773, 2, 9, 1, 9078, 180, 265, 31228] avg=8192.0000 max=31228 max_rank=7 min=1 min_rank=3 imbalance_ratio=3.812012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[9078, 9079, 4005, 9078, 9078, 9078, 7062, 9078] avg=8192.0000 max=9079 max_rank=1 min=4005 min_rank=2 imbalance_ratio=1.108276 max_load_reduction=22149 max_load_reduction_pct=0.709267 moved_to_replicas=75187 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[24773, 2, 9, 1, 9078, 180, 265, 31228] avg=8192.0000 max=31228 max_rank=7 min=1 min_rank=3 imbalance_ratio=3.812012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[24773, 2, 9, 1, 9078, 180, 265, 31228] avg=8192.0000 max=31228 max_rank=7 min=1 min_rank=3 imbalance_ratio=3.812012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[32639, 1608, 61, 1154, 23, 1958, 28090, 3] avg=8192.0000 max=32639 max_rank=0 min=3 min_rank=7 imbalance_ratio=3.984253 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[9078, 9079, 4005, 9078, 9078, 9078, 7062, 9078] avg=8192.0000 max=9079 max_rank=1 min=4005 min_rank=2 imbalance_ratio=1.108276 max_load_reduction=22149 max_load_reduction_pct=0.709267 moved_to_replicas=75187 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[24773, 2, 9, 1, 9078, 180, 265, 31228] avg=8192.0000 max=31228 max_rank=7 min=1 min_rank=3 imbalance_ratio=3.812012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[24773, 2, 9, 1, 9078, 180, 265, 31228] avg=8192.0000 max=31228 max_rank=7 min=1 min_rank=3 imbalance_ratio=3.812012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[32639, 1608, 61, 1154, 23, 1958, 28090, 3] avg=8192.0000 max=32639 max_rank=0 min=3 min_rank=7 imbalance_ratio=3.984253 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[9078, 9079, 4005, 9078, 9078, 9078, 7062, 9078] avg=8192.0000 max=9079 max_rank=1 min=4005 min_rank=2 imbalance_ratio=1.108276 max_load_reduction=22149 max_load_reduction_pct=0.709267 moved_to_replicas=75187 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[8273, 8274, 7904, 8274, 8274, 7988, 8274, 8275] avg=8192.0000 max=8275 max_rank=7 min=7904 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24364 max_load_reduction_pct=0.746469 moved_to_replicas=86687 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[32639, 1608, 61, 1154, 23, 1958, 28090, 3] avg=8192.0000 max=32639 max_rank=0 min=3 min_rank=7 imbalance_ratio=3.984253 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[9078, 9079, 4005, 9078, 9078, 9078, 7062, 9078] avg=8192.0000 max=9079 max_rank=1 min=4005 min_rank=2 imbalance_ratio=1.108276 max_load_reduction=22149 max_load_reduction_pct=0.709267 moved_to_replicas=75187 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[9078, 9079, 4005, 9078, 9078, 9078, 7062, 9078] avg=8192.0000 max=9079 max_rank=1 min=4005 min_rank=2 imbalance_ratio=1.108276 max_load_reduction=22149 max_load_reduction_pct=0.709267 moved_to_replicas=75187 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[9078, 9079, 4005, 9078, 9078, 9078, 7062, 9078] avg=8192.0000 max=9079 max_rank=1 min=4005 min_rank=2 imbalance_ratio=1.108276 max_load_reduction=22149 max_load_reduction_pct=0.709267 moved_to_replicas=75187 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[8273, 8274, 7904, 8274, 8274, 7988, 8274, 8275] avg=8192.0000 max=8275 max_rank=7 min=7904 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24364 max_load_reduction_pct=0.746469 moved_to_replicas=86687 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[32639, 1608, 61, 1154, 23, 1958, 28090, 3] avg=8192.0000 max=32639 max_rank=0 min=3 min_rank=7 imbalance_ratio=3.984253 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[8273, 8274, 7904, 8274, 8274, 7988, 8274, 8275] avg=8192.0000 max=8275 max_rank=7 min=7904 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24364 max_load_reduction_pct=0.746469 moved_to_replicas=86687 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[32639, 1608, 61, 1154, 23, 1958, 28090, 3] avg=8192.0000 max=32639 max_rank=0 min=3 min_rank=7 imbalance_ratio=3.984253 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[4228, 48, 23989, 19, 5, 4, 6457, 30786] avg=8192.0000 max=30786 max_rank=7 min=4 min_rank=5 imbalance_ratio=3.758057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[8273, 8274, 7904, 8274, 8274, 7988, 8274, 8275] avg=8192.0000 max=8275 max_rank=7 min=7904 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24364 max_load_reduction_pct=0.746469 moved_to_replicas=86687 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[32639, 1608, 61, 1154, 23, 1958, 28090, 3] avg=8192.0000 max=32639 max_rank=0 min=3 min_rank=7 imbalance_ratio=3.984253 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[32639, 1608, 61, 1154, 23, 1958, 28090, 3] avg=8192.0000 max=32639 max_rank=0 min=3 min_rank=7 imbalance_ratio=3.984253 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[32639, 1608, 61, 1154, 23, 1958, 28090, 3] avg=8192.0000 max=32639 max_rank=0 min=3 min_rank=7 imbalance_ratio=3.984253 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[4228, 48, 23989, 19, 5, 4, 6457, 30786] avg=8192.0000 max=30786 max_rank=7 min=4 min_rank=5 imbalance_ratio=3.758057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[8273, 8274, 7904, 8274, 8274, 7988, 8274, 8275] avg=8192.0000 max=8275 max_rank=7 min=7904 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24364 max_load_reduction_pct=0.746469 moved_to_replicas=86687 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[8890, 8890, 8890, 4144, 8890, 8890, 8052, 8890] avg=8192.0000 max=8890 max_rank=0 min=4144 min_rank=3 imbalance_ratio=1.085205 max_load_reduction=21896 max_load_reduction_pct=0.711232 moved_to_replicas=73397 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[4228, 48, 23989, 19, 5, 4, 6457, 30786] avg=8192.0000 max=30786 max_rank=7 min=4 min_rank=5 imbalance_ratio=3.758057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[8273, 8274, 7904, 8274, 8274, 7988, 8274, 8275] avg=8192.0000 max=8275 max_rank=7 min=7904 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24364 max_load_reduction_pct=0.746469 moved_to_replicas=86687 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[8273, 8274, 7904, 8274, 8274, 7988, 8274, 8275] avg=8192.0000 max=8275 max_rank=7 min=7904 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24364 max_load_reduction_pct=0.746469 moved_to_replicas=86687 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[8273, 8274, 7904, 8274, 8274, 7988, 8274, 8275] avg=8192.0000 max=8275 max_rank=7 min=7904 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24364 max_load_reduction_pct=0.746469 moved_to_replicas=86687 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[8890, 8890, 8890, 4144, 8890, 8890, 8052, 8890] avg=8192.0000 max=8890 max_rank=0 min=4144 min_rank=3 imbalance_ratio=1.085205 max_load_reduction=21896 max_load_reduction_pct=0.711232 moved_to_replicas=73397 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[4228, 48, 23989, 19, 5, 4, 6457, 30786] avg=8192.0000 max=30786 max_rank=7 min=4 min_rank=5 imbalance_ratio=3.758057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[8890, 8890, 8890, 4144, 8890, 8890, 8052, 8890] avg=8192.0000 max=8890 max_rank=0 min=4144 min_rank=3 imbalance_ratio=1.085205 max_load_reduction=21896 max_load_reduction_pct=0.711232 moved_to_replicas=73397 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[310, 9610, 31, 7, 5, 22883, 14, 32676] avg=8192.0000 max=32676 max_rank=7 min=5 min_rank=4 imbalance_ratio=3.988770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[4228, 48, 23989, 19, 5, 4, 6457, 30786] avg=8192.0000 max=30786 max_rank=7 min=4 min_rank=5 imbalance_ratio=3.758057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[8890, 8890, 8890, 4144, 8890, 8890, 8052, 8890] avg=8192.0000 max=8890 max_rank=0 min=4144 min_rank=3 imbalance_ratio=1.085205 max_load_reduction=21896 max_load_reduction_pct=0.711232 moved_to_replicas=73397 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[4228, 48, 23989, 19, 5, 4, 6457, 30786] avg=8192.0000 max=30786 max_rank=7 min=4 min_rank=5 imbalance_ratio=3.758057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[4228, 48, 23989, 19, 5, 4, 6457, 30786] avg=8192.0000 max=30786 max_rank=7 min=4 min_rank=5 imbalance_ratio=3.758057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[310, 9610, 31, 7, 5, 22883, 14, 32676] avg=8192.0000 max=32676 max_rank=7 min=5 min_rank=4 imbalance_ratio=3.988770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[4228, 48, 23989, 19, 5, 4, 6457, 30786] avg=8192.0000 max=30786 max_rank=7 min=4 min_rank=5 imbalance_ratio=3.758057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[4005, 9610, 9610, 9611, 9609, 9609, 3872, 9610] avg=8192.0000 max=9611 max_rank=3 min=3872 min_rank=6 imbalance_ratio=1.173218 max_load_reduction=23065 max_load_reduction_pct=0.705870 moved_to_replicas=71198 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[8890, 8890, 8890, 4144, 8890, 8890, 8052, 8890] avg=8192.0000 max=8890 max_rank=0 min=4144 min_rank=3 imbalance_ratio=1.085205 max_load_reduction=21896 max_load_reduction_pct=0.711232 moved_to_replicas=73397 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[310, 9610, 31, 7, 5, 22883, 14, 32676] avg=8192.0000 max=32676 max_rank=7 min=5 min_rank=4 imbalance_ratio=3.988770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[8890, 8890, 8890, 4144, 8890, 8890, 8052, 8890] avg=8192.0000 max=8890 max_rank=0 min=4144 min_rank=3 imbalance_ratio=1.085205 max_load_reduction=21896 max_load_reduction_pct=0.711232 moved_to_replicas=73397 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[8890, 8890, 8890, 4144, 8890, 8890, 8052, 8890] avg=8192.0000 max=8890 max_rank=0 min=4144 min_rank=3 imbalance_ratio=1.085205 max_load_reduction=21896 max_load_reduction_pct=0.711232 moved_to_replicas=73397 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[4005, 9610, 9610, 9611, 9609, 9609, 3872, 9610] avg=8192.0000 max=9611 max_rank=3 min=3872 min_rank=6 imbalance_ratio=1.173218 max_load_reduction=23065 max_load_reduction_pct=0.705870 moved_to_replicas=71198 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[8890, 8890, 8890, 4144, 8890, 8890, 8052, 8890] avg=8192.0000 max=8890 max_rank=0 min=4144 min_rank=3 imbalance_ratio=1.085205 max_load_reduction=21896 max_load_reduction_pct=0.711232 moved_to_replicas=73397 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[310, 9610, 31, 7, 5, 22883, 14, 32676] avg=8192.0000 max=32676 max_rank=7 min=5 min_rank=4 imbalance_ratio=3.988770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[29617, 11, 1464, 32615, 5, 1, 1823, 0] avg=8192.0000 max=32615 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[310, 9610, 31, 7, 5, 22883, 14, 32676] avg=8192.0000 max=32676 max_rank=7 min=5 min_rank=4 imbalance_ratio=3.988770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[4005, 9610, 9610, 9611, 9609, 9609, 3872, 9610] avg=8192.0000 max=9611 max_rank=3 min=3872 min_rank=6 imbalance_ratio=1.173218 max_load_reduction=23065 max_load_reduction_pct=0.705870 moved_to_replicas=71198 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[4005, 9610, 9610, 9611, 9609, 9609, 3872, 9610] avg=8192.0000 max=9611 max_rank=3 min=3872 min_rank=6 imbalance_ratio=1.173218 max_load_reduction=23065 max_load_reduction_pct=0.705870 moved_to_replicas=71198 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[310, 9610, 31, 7, 5, 22883, 14, 32676] avg=8192.0000 max=32676 max_rank=7 min=5 min_rank=4 imbalance_ratio=3.988770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[29617, 11, 1464, 32615, 5, 1, 1823, 0] avg=8192.0000 max=32615 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[310, 9610, 31, 7, 5, 22883, 14, 32676] avg=8192.0000 max=32676 max_rank=7 min=5 min_rank=4 imbalance_ratio=3.988770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[310, 9610, 31, 7, 5, 22883, 14, 32676] avg=8192.0000 max=32676 max_rank=7 min=5 min_rank=4 imbalance_ratio=3.988770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[8274, 8273, 8274, 8274, 7800, 8274, 8094, 8273] avg=8192.0000 max=8274 max_rank=0 min=7800 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24341 max_load_reduction_pct=0.746313 moved_to_replicas=91833 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[4005, 9610, 9610, 9611, 9609, 9609, 3872, 9610] avg=8192.0000 max=9611 max_rank=3 min=3872 min_rank=6 imbalance_ratio=1.173218 max_load_reduction=23065 max_load_reduction_pct=0.705870 moved_to_replicas=71198 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[4005, 9610, 9610, 9611, 9609, 9609, 3872, 9610] avg=8192.0000 max=9611 max_rank=3 min=3872 min_rank=6 imbalance_ratio=1.173218 max_load_reduction=23065 max_load_reduction_pct=0.705870 moved_to_replicas=71198 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[29617, 11, 1464, 32615, 5, 1, 1823, 0] avg=8192.0000 max=32615 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[8274, 8273, 8274, 8274, 7800, 8274, 8094, 8273] avg=8192.0000 max=8274 max_rank=0 min=7800 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24341 max_load_reduction_pct=0.746313 moved_to_replicas=91833 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[29617, 11, 1464, 32615, 5, 1, 1823, 0] avg=8192.0000 max=32615 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[4005, 9610, 9610, 9611, 9609, 9609, 3872, 9610] avg=8192.0000 max=9611 max_rank=3 min=3872 min_rank=6 imbalance_ratio=1.173218 max_load_reduction=23065 max_load_reduction_pct=0.705870 moved_to_replicas=71198 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[4005, 9610, 9610, 9611, 9609, 9609, 3872, 9610] avg=8192.0000 max=9611 max_rank=3 min=3872 min_rank=6 imbalance_ratio=1.173218 max_load_reduction=23065 max_load_reduction_pct=0.705870 moved_to_replicas=71198 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[16, 7, 2, 32726, 33, 4, 32743, 5] avg=8192.0000 max=32743 max_rank=6 min=2 min_rank=2 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[29617, 11, 1464, 32615, 5, 1, 1823, 0] avg=8192.0000 max=32615 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[8274, 8273, 8274, 8274, 7800, 8274, 8094, 8273] avg=8192.0000 max=8274 max_rank=0 min=7800 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24341 max_load_reduction_pct=0.746313 moved_to_replicas=91833 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[29617, 11, 1464, 32615, 5, 1, 1823, 0] avg=8192.0000 max=32615 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[8274, 8273, 8274, 8274, 7800, 8274, 8094, 8273] avg=8192.0000 max=8274 max_rank=0 min=7800 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24341 max_load_reduction_pct=0.746313 moved_to_replicas=91833 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[16, 7, 2, 32726, 33, 4, 32743, 5] avg=8192.0000 max=32743 max_rank=6 min=2 min_rank=2 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[29617, 11, 1464, 32615, 5, 1, 1823, 0] avg=8192.0000 max=32615 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[8274, 8273, 8274, 8274, 7961, 8274, 8274, 7932] avg=8192.0000 max=8274 max_rank=0 min=7932 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=24469 max_load_reduction_pct=0.747305 moved_to_replicas=97783 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[29617, 11, 1464, 32615, 5, 1, 1823, 0] avg=8192.0000 max=32615 max_rank=3 min=0 min_rank=7 imbalance_ratio=3.981323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[8274, 8273, 8274, 8274, 7800, 8274, 8094, 8273] avg=8192.0000 max=8274 max_rank=0 min=7800 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24341 max_load_reduction_pct=0.746313 moved_to_replicas=91833 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[8274, 8273, 8274, 8274, 7800, 8274, 8094, 8273] avg=8192.0000 max=8274 max_rank=0 min=7800 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24341 max_load_reduction_pct=0.746313 moved_to_replicas=91833 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[8274, 8273, 8274, 8274, 7961, 8274, 8274, 7932] avg=8192.0000 max=8274 max_rank=0 min=7932 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=24469 max_load_reduction_pct=0.747305 moved_to_replicas=97783 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[16, 7, 2, 32726, 33, 4, 32743, 5] avg=8192.0000 max=32743 max_rank=6 min=2 min_rank=2 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[8274, 8273, 8274, 8274, 7800, 8274, 8094, 8273] avg=8192.0000 max=8274 max_rank=0 min=7800 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24341 max_load_reduction_pct=0.746313 moved_to_replicas=91833 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[8274, 8273, 8274, 8274, 7800, 8274, 8094, 8273] avg=8192.0000 max=8274 max_rank=0 min=7800 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24341 max_load_reduction_pct=0.746313 moved_to_replicas=91833 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[16, 7, 2, 32726, 33, 4, 32743, 5] avg=8192.0000 max=32743 max_rank=6 min=2 min_rank=2 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[3, 7113, 6, 32747, 16215, 358, 13, 9081] avg=8192.0000 max=32747 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.997437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[16, 7, 2, 32726, 33, 4, 32743, 5] avg=8192.0000 max=32743 max_rank=6 min=2 min_rank=2 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[16, 7, 2, 32726, 33, 4, 32743, 5] avg=8192.0000 max=32743 max_rank=6 min=2 min_rank=2 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[8274, 8273, 8274, 8274, 7961, 8274, 8274, 7932] avg=8192.0000 max=8274 max_rank=0 min=7932 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=24469 max_load_reduction_pct=0.747305 moved_to_replicas=97783 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[8274, 8273, 8274, 8274, 7961, 8274, 8274, 7932] avg=8192.0000 max=8274 max_rank=0 min=7932 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=24469 max_load_reduction_pct=0.747305 moved_to_replicas=97783 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[8602, 7592, 8601, 8602, 8603, 7970, 6964, 8602] avg=8192.0000 max=8603 max_rank=4 min=6964 min_rank=6 imbalance_ratio=1.050171 max_load_reduction=24144 max_load_reduction_pct=0.737289 moved_to_replicas=66544 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[8274, 8273, 8274, 8274, 7961, 8274, 8274, 7932] avg=8192.0000 max=8274 max_rank=0 min=7932 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=24469 max_load_reduction_pct=0.747305 moved_to_replicas=97783 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[16, 7, 2, 32726, 33, 4, 32743, 5] avg=8192.0000 max=32743 max_rank=6 min=2 min_rank=2 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[16, 7, 2, 32726, 33, 4, 32743, 5] avg=8192.0000 max=32743 max_rank=6 min=2 min_rank=2 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[3, 7113, 6, 32747, 16215, 358, 13, 9081] avg=8192.0000 max=32747 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.997437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[8274, 8273, 8274, 8274, 7961, 8274, 8274, 7932] avg=8192.0000 max=8274 max_rank=0 min=7932 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=24469 max_load_reduction_pct=0.747305 moved_to_replicas=97783 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[3, 7113, 6, 32747, 16215, 358, 13, 9081] avg=8192.0000 max=32747 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.997437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[8274, 8273, 8274, 8274, 7961, 8274, 8274, 7932] avg=8192.0000 max=8274 max_rank=0 min=7932 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=24469 max_load_reduction_pct=0.747305 moved_to_replicas=97783 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[3, 7113, 6, 32747, 16215, 358, 13, 9081] avg=8192.0000 max=32747 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.997437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[8602, 7592, 8601, 8602, 8603, 7970, 6964, 8602] avg=8192.0000 max=8603 max_rank=4 min=6964 min_rank=6 imbalance_ratio=1.050171 max_load_reduction=24144 max_load_reduction_pct=0.737289 moved_to_replicas=66544 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[8274, 8273, 8274, 8274, 7961, 8274, 8274, 7932] avg=8192.0000 max=8274 max_rank=0 min=7932 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=24469 max_load_reduction_pct=0.747305 moved_to_replicas=97783 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[32716, 5, 7, 7, 17, 39, 32738, 7] avg=8192.0000 max=32738 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[3, 7113, 6, 32747, 16215, 358, 13, 9081] avg=8192.0000 max=32747 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.997437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[3, 7113, 6, 32747, 16215, 358, 13, 9081] avg=8192.0000 max=32747 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.997437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[8602, 7592, 8601, 8602, 8603, 7970, 6964, 8602] avg=8192.0000 max=8603 max_rank=4 min=6964 min_rank=6 imbalance_ratio=1.050171 max_load_reduction=24144 max_load_reduction_pct=0.737289 moved_to_replicas=66544 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[8602, 7592, 8601, 8602, 8603, 7970, 6964, 8602] avg=8192.0000 max=8603 max_rank=4 min=6964 min_rank=6 imbalance_ratio=1.050171 max_load_reduction=24144 max_load_reduction_pct=0.737289 moved_to_replicas=66544 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[8273, 8274, 8274, 7934, 8274, 7957, 8275, 8275] avg=8192.0000 max=8275 max_rank=6 min=7934 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=97754 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[8602, 7592, 8601, 8602, 8603, 7970, 6964, 8602] avg=8192.0000 max=8603 max_rank=4 min=6964 min_rank=6 imbalance_ratio=1.050171 max_load_reduction=24144 max_load_reduction_pct=0.737289 moved_to_replicas=66544 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[8602, 7592, 8601, 8602, 8603, 7970, 6964, 8602] avg=8192.0000 max=8603 max_rank=4 min=6964 min_rank=6 imbalance_ratio=1.050171 max_load_reduction=24144 max_load_reduction_pct=0.737289 moved_to_replicas=66544 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[3, 7113, 6, 32747, 16215, 358, 13, 9081] avg=8192.0000 max=32747 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.997437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[32716, 5, 7, 7, 17, 39, 32738, 7] avg=8192.0000 max=32738 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[3, 7113, 6, 32747, 16215, 358, 13, 9081] avg=8192.0000 max=32747 max_rank=3 min=3 min_rank=0 imbalance_ratio=3.997437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[32716, 5, 7, 7, 17, 39, 32738, 7] avg=8192.0000 max=32738 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[32716, 5, 7, 7, 17, 39, 32738, 7] avg=8192.0000 max=32738 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[8602, 7592, 8601, 8602, 8603, 7970, 6964, 8602] avg=8192.0000 max=8603 max_rank=4 min=6964 min_rank=6 imbalance_ratio=1.050171 max_load_reduction=24144 max_load_reduction_pct=0.737289 moved_to_replicas=66544 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[8273, 8274, 8274, 7934, 8274, 7957, 8275, 8275] avg=8192.0000 max=8275 max_rank=6 min=7934 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=97754 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[9, 17210, 32740, 3, 8, 2, 15544, 20] avg=8192.0000 max=32740 max_rank=2 min=2 min_rank=5 imbalance_ratio=3.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[8602, 7592, 8601, 8602, 8603, 7970, 6964, 8602] avg=8192.0000 max=8603 max_rank=4 min=6964 min_rank=6 imbalance_ratio=1.050171 max_load_reduction=24144 max_load_reduction_pct=0.737289 moved_to_replicas=66544 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[32716, 5, 7, 7, 17, 39, 32738, 7] avg=8192.0000 max=32738 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[32716, 5, 7, 7, 17, 39, 32738, 7] avg=8192.0000 max=32738 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[8273, 8274, 8274, 7934, 8274, 7957, 8275, 8275] avg=8192.0000 max=8275 max_rank=6 min=7934 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=97754 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[8273, 8274, 8274, 7934, 8274, 7957, 8275, 8275] avg=8192.0000 max=8275 max_rank=6 min=7934 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=97754 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[8289, 8930, 8931, 8930, 5962, 8930, 8931, 6633] avg=8192.0000 max=8931 max_rank=2 min=5962 min_rank=4 imbalance_ratio=1.090210 max_load_reduction=23809 max_load_reduction_pct=0.727214 moved_to_replicas=78381 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[8273, 8274, 8274, 7934, 8274, 7957, 8275, 8275] avg=8192.0000 max=8275 max_rank=6 min=7934 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=97754 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[8273, 8274, 8274, 7934, 8274, 7957, 8275, 8275] avg=8192.0000 max=8275 max_rank=6 min=7934 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=97754 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[32716, 5, 7, 7, 17, 39, 32738, 7] avg=8192.0000 max=32738 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[9, 17210, 32740, 3, 8, 2, 15544, 20] avg=8192.0000 max=32740 max_rank=2 min=2 min_rank=5 imbalance_ratio=3.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[32716, 5, 7, 7, 17, 39, 32738, 7] avg=8192.0000 max=32738 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[9, 17210, 32740, 3, 8, 2, 15544, 20] avg=8192.0000 max=32740 max_rank=2 min=2 min_rank=5 imbalance_ratio=3.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[9, 17210, 32740, 3, 8, 2, 15544, 20] avg=8192.0000 max=32740 max_rank=2 min=2 min_rank=5 imbalance_ratio=3.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[8273, 8274, 8274, 7934, 8274, 7957, 8275, 8275] avg=8192.0000 max=8275 max_rank=6 min=7934 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=97754 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[8289, 8930, 8931, 8930, 5962, 8930, 8931, 6633] avg=8192.0000 max=8931 max_rank=2 min=5962 min_rank=4 imbalance_ratio=1.090210 max_load_reduction=23809 max_load_reduction_pct=0.727214 moved_to_replicas=78381 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[8273, 8274, 8274, 7934, 8274, 7957, 8275, 8275] avg=8192.0000 max=8275 max_rank=6 min=7934 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=97754 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[8289, 8930, 8931, 8930, 5962, 8930, 8931, 6633] avg=8192.0000 max=8931 max_rank=2 min=5962 min_rank=4 imbalance_ratio=1.090210 max_load_reduction=23809 max_load_reduction_pct=0.727214 moved_to_replicas=78381 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[9, 17210, 32740, 3, 8, 2, 15544, 20] avg=8192.0000 max=32740 max_rank=2 min=2 min_rank=5 imbalance_ratio=3.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[9, 17210, 32740, 3, 8, 2, 15544, 20] avg=8192.0000 max=32740 max_rank=2 min=2 min_rank=5 imbalance_ratio=3.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[8289, 8930, 8931, 8930, 5962, 8930, 8931, 6633] avg=8192.0000 max=8931 max_rank=2 min=5962 min_rank=4 imbalance_ratio=1.090210 max_load_reduction=23809 max_load_reduction_pct=0.727214 moved_to_replicas=78381 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[18, 12, 32635, 32741, 2, 9, 15, 104] avg=8192.0000 max=32741 max_rank=3 min=2 min_rank=4 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[8289, 8930, 8931, 8930, 5962, 8930, 8931, 6633] avg=8192.0000 max=8931 max_rank=2 min=5962 min_rank=4 imbalance_ratio=1.090210 max_load_reduction=23809 max_load_reduction_pct=0.727214 moved_to_replicas=78381 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[18, 12, 32635, 32741, 2, 9, 15, 104] avg=8192.0000 max=32741 max_rank=3 min=2 min_rank=4 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[9, 17210, 32740, 3, 8, 2, 15544, 20] avg=8192.0000 max=32740 max_rank=2 min=2 min_rank=5 imbalance_ratio=3.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[8289, 8930, 8931, 8930, 5962, 8930, 8931, 6633] avg=8192.0000 max=8931 max_rank=2 min=5962 min_rank=4 imbalance_ratio=1.090210 max_load_reduction=23809 max_load_reduction_pct=0.727214 moved_to_replicas=78381 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[9, 17210, 32740, 3, 8, 2, 15544, 20] avg=8192.0000 max=32740 max_rank=2 min=2 min_rank=5 imbalance_ratio=3.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[18, 12, 32635, 32741, 2, 9, 15, 104] avg=8192.0000 max=32741 max_rank=3 min=2 min_rank=4 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[8275, 7942, 8275, 8273, 8275, 8274, 8273, 7949] avg=8192.0000 max=8275 max_rank=0 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24466 max_load_reduction_pct=0.747259 moved_to_replicas=97621 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[18, 12, 32635, 32741, 2, 9, 15, 104] avg=8192.0000 max=32741 max_rank=3 min=2 min_rank=4 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[8275, 7942, 8275, 8273, 8275, 8274, 8273, 7949] avg=8192.0000 max=8275 max_rank=0 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24466 max_load_reduction_pct=0.747259 moved_to_replicas=97621 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[8289, 8930, 8931, 8930, 5962, 8930, 8931, 6633] avg=8192.0000 max=8931 max_rank=2 min=5962 min_rank=4 imbalance_ratio=1.090210 max_load_reduction=23809 max_load_reduction_pct=0.727214 moved_to_replicas=78381 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[8275, 7942, 8275, 8273, 8275, 8274, 8273, 7949] avg=8192.0000 max=8275 max_rank=0 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24466 max_load_reduction_pct=0.747259 moved_to_replicas=97621 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[8289, 8930, 8931, 8930, 5962, 8930, 8931, 6633] avg=8192.0000 max=8931 max_rank=2 min=5962 min_rank=4 imbalance_ratio=1.090210 max_load_reduction=23809 max_load_reduction_pct=0.727214 moved_to_replicas=78381 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[18, 12, 32635, 32741, 2, 9, 15, 104] avg=8192.0000 max=32741 max_rank=3 min=2 min_rank=4 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[18, 12, 32635, 32741, 2, 9, 15, 104] avg=8192.0000 max=32741 max_rank=3 min=2 min_rank=4 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[8275, 7942, 8275, 8273, 8275, 8274, 8273, 7949] avg=8192.0000 max=8275 max_rank=0 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24466 max_load_reduction_pct=0.747259 moved_to_replicas=97621 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[17459, 15235, 17, 39, 32733, 23, 16, 14] avg=8192.0000 max=32733 max_rank=4 min=14 min_rank=7 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[8275, 7942, 8275, 8273, 8275, 8274, 8273, 7949] avg=8192.0000 max=8275 max_rank=0 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24466 max_load_reduction_pct=0.747259 moved_to_replicas=97621 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[17459, 15235, 17, 39, 32733, 23, 16, 14] avg=8192.0000 max=32733 max_rank=4 min=14 min_rank=7 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[18, 12, 32635, 32741, 2, 9, 15, 104] avg=8192.0000 max=32741 max_rank=3 min=2 min_rank=4 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[8275, 7942, 8275, 8273, 8275, 8274, 8273, 7949] avg=8192.0000 max=8275 max_rank=0 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24466 max_load_reduction_pct=0.747259 moved_to_replicas=97621 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[17459, 15235, 17, 39, 32733, 23, 16, 14] avg=8192.0000 max=32733 max_rank=4 min=14 min_rank=7 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[18, 12, 32635, 32741, 2, 9, 15, 104] avg=8192.0000 max=32741 max_rank=3 min=2 min_rank=4 imbalance_ratio=3.996704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[8931, 8930, 5990, 6344, 8930, 8551, 8929, 8931] avg=8192.0000 max=8931 max_rank=0 min=5990 min_rank=2 imbalance_ratio=1.090210 max_load_reduction=23802 max_load_reduction_pct=0.727156 moved_to_replicas=74506 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[8931, 8930, 5990, 6344, 8930, 8551, 8929, 8931] avg=8192.0000 max=8931 max_rank=0 min=5990 min_rank=2 imbalance_ratio=1.090210 max_load_reduction=23802 max_load_reduction_pct=0.727156 moved_to_replicas=74506 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[17459, 15235, 17, 39, 32733, 23, 16, 14] avg=8192.0000 max=32733 max_rank=4 min=14 min_rank=7 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[8275, 7942, 8275, 8273, 8275, 8274, 8273, 7949] avg=8192.0000 max=8275 max_rank=0 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24466 max_load_reduction_pct=0.747259 moved_to_replicas=97621 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[8931, 8930, 5990, 6344, 8930, 8551, 8929, 8931] avg=8192.0000 max=8931 max_rank=0 min=5990 min_rank=2 imbalance_ratio=1.090210 max_load_reduction=23802 max_load_reduction_pct=0.727156 moved_to_replicas=74506 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[8275, 7942, 8275, 8273, 8275, 8274, 8273, 7949] avg=8192.0000 max=8275 max_rank=0 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24466 max_load_reduction_pct=0.747259 moved_to_replicas=97621 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[17459, 15235, 17, 39, 32733, 23, 16, 14] avg=8192.0000 max=32733 max_rank=4 min=14 min_rank=7 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[17459, 15235, 17, 39, 32733, 23, 16, 14] avg=8192.0000 max=32733 max_rank=4 min=14 min_rank=7 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[0, 21, 6890, 25853, 6, 21, 7, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[8931, 8930, 5990, 6344, 8930, 8551, 8929, 8931] avg=8192.0000 max=8931 max_rank=0 min=5990 min_rank=2 imbalance_ratio=1.090210 max_load_reduction=23802 max_load_reduction_pct=0.727156 moved_to_replicas=74506 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[8931, 8930, 5990, 6344, 8930, 8551, 8929, 8931] avg=8192.0000 max=8931 max_rank=0 min=5990 min_rank=2 imbalance_ratio=1.090210 max_load_reduction=23802 max_load_reduction_pct=0.727156 moved_to_replicas=74506 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[0, 21, 6890, 25853, 6, 21, 7, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[17459, 15235, 17, 39, 32733, 23, 16, 14] avg=8192.0000 max=32733 max_rank=4 min=14 min_rank=7 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[0, 21, 6890, 25853, 6, 21, 7, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[8931, 8930, 5990, 6344, 8930, 8551, 8929, 8931] avg=8192.0000 max=8931 max_rank=0 min=5990 min_rank=2 imbalance_ratio=1.090210 max_load_reduction=23802 max_load_reduction_pct=0.727156 moved_to_replicas=74506 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[17459, 15235, 17, 39, 32733, 23, 16, 14] avg=8192.0000 max=32733 max_rank=4 min=14 min_rank=7 imbalance_ratio=3.995728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[8273, 8274, 7963, 8274, 8274, 8274, 7930, 8274] avg=8192.0000 max=8274 max_rank=1 min=7930 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24464 max_load_reduction_pct=0.747266 moved_to_replicas=85362 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[8273, 8274, 7963, 8274, 8274, 8274, 7930, 8274] avg=8192.0000 max=8274 max_rank=1 min=7930 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24464 max_load_reduction_pct=0.747266 moved_to_replicas=85362 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[8931, 8930, 5990, 6344, 8930, 8551, 8929, 8931] avg=8192.0000 max=8931 max_rank=0 min=5990 min_rank=2 imbalance_ratio=1.090210 max_load_reduction=23802 max_load_reduction_pct=0.727156 moved_to_replicas=74506 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[0, 21, 6890, 25853, 6, 21, 7, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[8273, 8274, 7963, 8274, 8274, 8274, 7930, 8274] avg=8192.0000 max=8274 max_rank=1 min=7930 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24464 max_load_reduction_pct=0.747266 moved_to_replicas=85362 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[0, 21, 6890, 25853, 6, 21, 7, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[8931, 8930, 5990, 6344, 8930, 8551, 8929, 8931] avg=8192.0000 max=8931 max_rank=0 min=5990 min_rank=2 imbalance_ratio=1.090210 max_load_reduction=23802 max_load_reduction_pct=0.727156 moved_to_replicas=74506 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[0, 21, 6890, 25853, 6, 21, 7, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[13, 30559, 0, 23, 2175, 8, 13, 32745] avg=8192.0000 max=32745 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[8273, 8274, 7963, 8274, 8274, 8274, 7930, 8274] avg=8192.0000 max=8274 max_rank=1 min=7930 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24464 max_load_reduction_pct=0.747266 moved_to_replicas=85362 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[8273, 8274, 7963, 8274, 8274, 8274, 7930, 8274] avg=8192.0000 max=8274 max_rank=1 min=7930 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24464 max_load_reduction_pct=0.747266 moved_to_replicas=85362 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[13, 30559, 0, 23, 2175, 8, 13, 32745] avg=8192.0000 max=32745 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[0, 21, 6890, 25853, 6, 21, 7, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[13, 30559, 0, 23, 2175, 8, 13, 32745] avg=8192.0000 max=32745 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[8273, 8274, 7963, 8274, 8274, 8274, 7930, 8274] avg=8192.0000 max=8274 max_rank=1 min=7930 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24464 max_load_reduction_pct=0.747266 moved_to_replicas=85362 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[0, 21, 6890, 25853, 6, 21, 7, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[7942, 8274, 8275, 8274, 7948, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=2 min=7942 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24470 max_load_reduction_pct=0.747290 moved_to_replicas=93654 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[7942, 8274, 8275, 8274, 7948, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=2 min=7942 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24470 max_load_reduction_pct=0.747290 moved_to_replicas=93654 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[8273, 8274, 7963, 8274, 8274, 8274, 7930, 8274] avg=8192.0000 max=8274 max_rank=1 min=7930 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24464 max_load_reduction_pct=0.747266 moved_to_replicas=85362 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[7942, 8274, 8275, 8274, 7948, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=2 min=7942 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24470 max_load_reduction_pct=0.747290 moved_to_replicas=93654 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[13, 30559, 0, 23, 2175, 8, 13, 32745] avg=8192.0000 max=32745 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[13, 30559, 0, 23, 2175, 8, 13, 32745] avg=8192.0000 max=32745 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[8273, 8274, 7963, 8274, 8274, 8274, 7930, 8274] avg=8192.0000 max=8274 max_rank=1 min=7930 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=24464 max_load_reduction_pct=0.747266 moved_to_replicas=85362 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[26, 32739, 32684, 0, 18, 12, 2, 55] avg=8192.0000 max=32739 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[13, 30559, 0, 23, 2175, 8, 13, 32745] avg=8192.0000 max=32745 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[7942, 8274, 8275, 8274, 7948, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=2 min=7942 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24470 max_load_reduction_pct=0.747290 moved_to_replicas=93654 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[7942, 8274, 8275, 8274, 7948, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=2 min=7942 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24470 max_load_reduction_pct=0.747290 moved_to_replicas=93654 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[26, 32739, 32684, 0, 18, 12, 2, 55] avg=8192.0000 max=32739 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[13, 30559, 0, 23, 2175, 8, 13, 32745] avg=8192.0000 max=32745 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[26, 32739, 32684, 0, 18, 12, 2, 55] avg=8192.0000 max=32739 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[8273, 8274, 8275, 8274, 8273, 7931, 8274, 7962] avg=8192.0000 max=8275 max_rank=2 min=7931 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24464 max_load_reduction_pct=0.747243 moved_to_replicas=97695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[13, 30559, 0, 23, 2175, 8, 13, 32745] avg=8192.0000 max=32745 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[7942, 8274, 8275, 8274, 7948, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=2 min=7942 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24470 max_load_reduction_pct=0.747290 moved_to_replicas=93654 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[8273, 8274, 8275, 8274, 8273, 7931, 8274, 7962] avg=8192.0000 max=8275 max_rank=2 min=7931 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24464 max_load_reduction_pct=0.747243 moved_to_replicas=97695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[7942, 8274, 8275, 8274, 7948, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=2 min=7942 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24470 max_load_reduction_pct=0.747290 moved_to_replicas=93654 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[8273, 8274, 8275, 8274, 8273, 7931, 8274, 7962] avg=8192.0000 max=8275 max_rank=2 min=7931 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24464 max_load_reduction_pct=0.747243 moved_to_replicas=97695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[26, 32739, 32684, 0, 18, 12, 2, 55] avg=8192.0000 max=32739 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[26, 32739, 32684, 0, 18, 12, 2, 55] avg=8192.0000 max=32739 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[7942, 8274, 8275, 8274, 7948, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=2 min=7942 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24470 max_load_reduction_pct=0.747290 moved_to_replicas=93654 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[6, 32706, 25397, 4, 21, 22, 7369, 11] avg=8192.0000 max=32706 max_rank=1 min=4 min_rank=3 imbalance_ratio=3.992432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[8273, 8274, 8275, 8274, 8273, 7931, 8274, 7962] avg=8192.0000 max=8275 max_rank=2 min=7931 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24464 max_load_reduction_pct=0.747243 moved_to_replicas=97695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[26, 32739, 32684, 0, 18, 12, 2, 55] avg=8192.0000 max=32739 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[6, 32706, 25397, 4, 21, 22, 7369, 11] avg=8192.0000 max=32706 max_rank=1 min=4 min_rank=3 imbalance_ratio=3.992432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[8273, 8274, 8275, 8274, 8273, 7931, 8274, 7962] avg=8192.0000 max=8275 max_rank=2 min=7931 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24464 max_load_reduction_pct=0.747243 moved_to_replicas=97695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[26, 32739, 32684, 0, 18, 12, 2, 55] avg=8192.0000 max=32739 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[6, 32706, 25397, 4, 21, 22, 7369, 11] avg=8192.0000 max=32706 max_rank=1 min=4 min_rank=3 imbalance_ratio=3.992432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7987, 7904] avg=8192.0000 max=8275 max_rank=1 min=7904 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24431 max_load_reduction_pct=0.746988 moved_to_replicas=83650 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[26, 32739, 32684, 0, 18, 12, 2, 55] avg=8192.0000 max=32739 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.996460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[8273, 8274, 8275, 8274, 8273, 7931, 8274, 7962] avg=8192.0000 max=8275 max_rank=2 min=7931 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24464 max_load_reduction_pct=0.747243 moved_to_replicas=97695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7987, 7904] avg=8192.0000 max=8275 max_rank=1 min=7904 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24431 max_load_reduction_pct=0.746988 moved_to_replicas=83650 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[8273, 8274, 8275, 8274, 8273, 7931, 8274, 7962] avg=8192.0000 max=8275 max_rank=2 min=7931 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24464 max_load_reduction_pct=0.747243 moved_to_replicas=97695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7987, 7904] avg=8192.0000 max=8275 max_rank=1 min=7904 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24431 max_load_reduction_pct=0.746988 moved_to_replicas=83650 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[6, 32706, 25397, 4, 21, 22, 7369, 11] avg=8192.0000 max=32706 max_rank=1 min=4 min_rank=3 imbalance_ratio=3.992432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[6, 32706, 25397, 4, 21, 22, 7369, 11] avg=8192.0000 max=32706 max_rank=1 min=4 min_rank=3 imbalance_ratio=3.992432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[8273, 8274, 8275, 8274, 8273, 7931, 8274, 7962] avg=8192.0000 max=8275 max_rank=2 min=7931 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24464 max_load_reduction_pct=0.747243 moved_to_replicas=97695 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[6, 32706, 25397, 4, 21, 22, 7369, 11] avg=8192.0000 max=32706 max_rank=1 min=4 min_rank=3 imbalance_ratio=3.992432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7987, 7904] avg=8192.0000 max=8275 max_rank=1 min=7904 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24431 max_load_reduction_pct=0.746988 moved_to_replicas=83650 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7987, 7904] avg=8192.0000 max=8275 max_rank=1 min=7904 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24431 max_load_reduction_pct=0.746988 moved_to_replicas=83650 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[6, 32706, 25397, 4, 21, 22, 7369, 11] avg=8192.0000 max=32706 max_rank=1 min=4 min_rank=3 imbalance_ratio=3.992432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[6, 32706, 25397, 4, 21, 22, 7369, 11] avg=8192.0000 max=32706 max_rank=1 min=4 min_rank=3 imbalance_ratio=3.992432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7987, 7904] avg=8192.0000 max=8275 max_rank=1 min=7904 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24431 max_load_reduction_pct=0.746988 moved_to_replicas=83650 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7987, 7904] avg=8192.0000 max=8275 max_rank=1 min=7904 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24431 max_load_reduction_pct=0.746988 moved_to_replicas=83650 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7987, 7904] avg=8192.0000 max=8275 max_rank=1 min=7904 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24431 max_load_reduction_pct=0.746988 moved_to_replicas=83650 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=32 total_expected_recv=252578 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=219008 peak_demand=9998 required_capacity=1.220459 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=32 total_expected_recv=266057 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=204160 peak_demand=9996 required_capacity=1.220215 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=32 total_expected_recv=263927 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=207104 peak_demand=9997 required_capacity=1.220337 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=32 total_expected_recv=269155 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=202368 peak_demand=9997 required_capacity=1.220337 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=32 total_expected_recv=269317 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=201344 peak_demand=9997 required_capacity=1.220337 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=32 total_expected_recv=272547 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=199040 peak_demand=9997 required_capacity=1.220337 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=32 total_expected_recv=244776 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=226176 peak_demand=8931 required_capacity=1.090210 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=32 total_expected_recv=258795 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=212352 peak_demand=9855 required_capacity=1.203003 + [2026-08-30 05:09:13] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 928.2 | throughput per GPU (TFLOP/s/GPU): 359.2 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.119576E+01 | loss scale: 1.0 | grad norm: 89.108 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 13312 | min safe capacity: 1.6250 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=14848 expected_recv=20260 expected_overflow=0 exact_recv_per_rank=[7561, 7611, 5178, 9036, 9038, 9037, 9038, 9037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=14848 expected_recv=7810 expected_overflow=0 exact_recv_per_rank=[7561, 7611, 5178, 9036, 9038, 9037, 9038, 9037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=14848 expected_recv=10409 expected_overflow=0 exact_recv_per_rank=[7561, 7611, 5178, 9036, 9038, 9037, 9038, 9037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=14848 expected_recv=13778 expected_overflow=0 exact_recv_per_rank=[7561, 7611, 5178, 9036, 9038, 9037, 9038, 9037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=14848 expected_recv=20262 expected_overflow=0 exact_recv_per_rank=[7561, 7611, 5178, 9036, 9038, 9037, 9038, 9037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=14848 expected_recv=20261 expected_overflow=0 exact_recv_per_rank=[7561, 7611, 5178, 9036, 9038, 9037, 9038, 9037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=14848 expected_recv=20262 expected_overflow=0 exact_recv_per_rank=[7561, 7611, 5178, 9036, 9038, 9037, 9038, 9037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=14848 expected_recv=20260 actual_valid_recv=20352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=14848 expected_recv=7810 actual_valid_recv=8064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=14848 expected_recv=18030 expected_overflow=0 exact_recv_per_rank=[7561, 7611, 5178, 9036, 9038, 9037, 9038, 9037] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=14848 expected_recv=10409 actual_valid_recv=10752 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=14848 expected_recv=13778 actual_valid_recv=14208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=14848 expected_recv=20262 actual_valid_recv=20352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=20352 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=8064 slack_slots=21632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=14848 expected_recv=20261 actual_valid_recv=20352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=14848 expected_recv=20262 actual_valid_recv=20352 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=10752 slack_slots=18944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=14208 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=20352 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=14848 expected_recv=18030 actual_valid_recv=18304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=20352 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=20352 slack_slots=9344 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=18304 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=14848 expected_recv=16793 expected_overflow=0 exact_recv_per_rank=[9060, 9058, 9059, 9058, 9060, 9059, 5063, 6119] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=14848 expected_recv=19185 expected_overflow=0 exact_recv_per_rank=[9060, 9058, 9059, 9058, 9060, 9059, 5063, 6119] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=14848 expected_recv=19418 expected_overflow=0 exact_recv_per_rank=[9060, 9058, 9059, 9058, 9060, 9059, 5063, 6119] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=14848 expected_recv=19420 expected_overflow=0 exact_recv_per_rank=[9060, 9058, 9059, 9058, 9060, 9059, 5063, 6119] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=14848 expected_recv=10720 expected_overflow=0 exact_recv_per_rank=[9060, 9058, 9059, 9058, 9060, 9059, 5063, 6119] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=14848 expected_recv=19419 expected_overflow=0 exact_recv_per_rank=[9060, 9058, 9059, 9058, 9060, 9059, 5063, 6119] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=14848 expected_recv=16793 actual_valid_recv=17152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=14848 expected_recv=19185 actual_valid_recv=19584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=14848 expected_recv=19418 actual_valid_recv=19712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=14848 expected_recv=6698 expected_overflow=0 exact_recv_per_rank=[9060, 9058, 9059, 9058, 9060, 9059, 5063, 6119] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=14848 expected_recv=19420 actual_valid_recv=19456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=14848 expected_recv=10720 actual_valid_recv=10880 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=14848 expected_recv=19419 actual_valid_recv=19456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=19584 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=14848 expected_recv=19419 expected_overflow=0 exact_recv_per_rank=[9060, 9058, 9059, 9058, 9060, 9059, 5063, 6119] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=17152 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=19712 slack_slots=9984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=19456 slack_slots=10240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=10880 slack_slots=18816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=14848 expected_recv=6698 actual_valid_recv=6784 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=19456 slack_slots=10240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=14848 expected_recv=19419 actual_valid_recv=19456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=6784 slack_slots=22912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=19456 slack_slots=10240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 7943, 8274, 8274, 8274, 7949, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 7943, 8274, 8274, 8274, 7949, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=14848 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8275, 7943, 8274, 8274, 8274, 7949, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=14848 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8275, 7943, 8274, 8274, 8274, 7949, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 7943, 8274, 8274, 8274, 7949, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=14848 expected_recv=16218 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=14848 expected_recv=15888 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=14848 expected_recv=16224 expected_overflow=0 exact_recv_per_rank=[8275, 7943, 8274, 8274, 8274, 7949, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 7943, 8274, 8274, 8274, 7949, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 7943, 8274, 8274, 8274, 7949, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=14848 expected_recv=16224 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8274, 8274, 7944, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=14848 expected_recv=16222 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8274, 8274, 7944, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8274, 8274, 7944, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8274, 8274, 7944, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=14848 expected_recv=16222 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8274, 8274, 7944, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8274, 8274, 7944, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8274, 8274, 7944, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8274, 8274, 7944, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=14848 expected_recv=16215 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8273, 8273, 8274, 8272, 7948, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=14848 expected_recv=16224 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8273, 8273, 8274, 8272, 7948, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=14848 expected_recv=16215 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=14848 expected_recv=16224 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8273, 8273, 8274, 8272, 7948, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8273, 8273, 8274, 8272, 7948, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8273, 8273, 8274, 8272, 7948, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=14848 expected_recv=16222 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8273, 8273, 8274, 8272, 7948, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=14848 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8273, 8273, 8274, 8272, 7948, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8273, 8273, 8274, 8272, 7948, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=14848 expected_recv=16222 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=14848 expected_recv=16545 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=14848 expected_recv=16221 expected_overflow=0 exact_recv_per_rank=[7949, 8275, 8274, 8275, 8275, 8274, 8274, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7949, 8275, 8274, 8275, 8275, 8274, 8274, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=14848 expected_recv=16221 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=14848 expected_recv=15884 expected_overflow=0 exact_recv_per_rank=[7949, 8275, 8274, 8275, 8275, 8274, 8274, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7949, 8275, 8274, 8275, 8275, 8274, 8274, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=14848 expected_recv=16223 expected_overflow=0 exact_recv_per_rank=[7949, 8275, 8274, 8275, 8275, 8274, 8274, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=14848 expected_recv=15884 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7949, 8275, 8274, 8275, 8275, 8274, 8274, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7949, 8275, 8274, 8275, 8275, 8274, 8274, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7949, 8275, 8274, 8275, 8275, 8274, 8274, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=14848 expected_recv=16223 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7949, 8274, 8276, 8276, 8273, 8274, 8272, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=14848 expected_recv=16214 expected_overflow=0 exact_recv_per_rank=[7949, 8274, 8276, 8276, 8273, 8274, 8272, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=14848 expected_recv=16215 expected_overflow=0 exact_recv_per_rank=[7949, 8274, 8276, 8276, 8273, 8274, 8272, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=14848 expected_recv=16214 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7949, 8274, 8276, 8276, 8273, 8274, 8272, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=14848 expected_recv=16215 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=14848 expected_recv=15905 expected_overflow=0 exact_recv_per_rank=[7949, 8274, 8276, 8276, 8273, 8274, 8272, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=14848 expected_recv=15905 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=14848 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[7949, 8274, 8276, 8276, 8273, 8274, 8272, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7949, 8274, 8276, 8276, 8273, 8274, 8272, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[7949, 8274, 8276, 8276, 8273, 8274, 8272, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=14848 expected_recv=16545 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8276, 7944, 8274, 7946, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=14848 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8276, 7944, 8274, 7946, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8276, 7944, 8274, 7946, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=14848 expected_recv=16217 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8276, 7944, 8274, 7946, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8276, 7944, 8274, 7946, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=14848 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8276, 7944, 8274, 7946, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8276, 7944, 8274, 7946, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=14848 expected_recv=15895 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8276, 7944, 8274, 7946, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=14848 expected_recv=16216 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=14848 expected_recv=15895 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=14848 expected_recv=16222 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=14848 expected_recv=16222 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=14848 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=14848 expected_recv=16214 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=14848 expected_recv=16220 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 7947, 8276, 8272, 7944, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=14848 expected_recv=16214 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 7947, 8276, 8272, 7944, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 7947, 8276, 8272, 7944, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=14848 expected_recv=15903 expected_overflow=0 exact_recv_per_rank=[8276, 7947, 8276, 8272, 7944, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 7947, 8276, 8272, 7944, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=14848 expected_recv=15903 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=14848 expected_recv=16213 expected_overflow=0 exact_recv_per_rank=[8276, 7947, 8276, 8272, 7944, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 7947, 8276, 8272, 7944, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=14848 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8276, 7947, 8276, 8272, 7944, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=14848 expected_recv=16213 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=14848 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8272, 8275, 8274, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=14848 expected_recv=16218 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=14848 expected_recv=16218 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8272, 8275, 8274, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=14848 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8272, 8275, 8274, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=14848 expected_recv=16545 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8272, 8275, 8274, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=14848 expected_recv=15884 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8272, 8275, 8274, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=14848 expected_recv=15884 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8272, 8275, 8274, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=14848 expected_recv=16230 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8272, 8275, 8274, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8272, 8275, 8274, 7947, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=14848 expected_recv=19527 expected_overflow=0 exact_recv_per_rank=[10264, 11254, 11255, 10261, 11251, 11251, 0, 0] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=14848 expected_recv=16230 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=14848 expected_recv=18536 expected_overflow=0 exact_recv_per_rank=[10264, 11254, 11255, 10261, 11251, 11251, 0, 0] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=14848 expected_recv=8274 expected_overflow=0 exact_recv_per_rank=[10264, 11254, 11255, 10261, 11251, 11251, 0, 0] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=14848 expected_recv=19527 actual_valid_recv=19712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=14848 expected_recv=18536 actual_valid_recv=18688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=14848 expected_recv=8274 actual_valid_recv=8448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=19712 slack_slots=9984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=18688 slack_slots=11008 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=14848 expected_recv=19529 expected_overflow=0 exact_recv_per_rank=[10264, 11254, 11255, 10261, 11251, 11251, 0, 0] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=8448 slack_slots=21248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=14848 expected_recv=7940 expected_overflow=0 exact_recv_per_rank=[10264, 11254, 11255, 10261, 11251, 11251, 0, 0] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=14848 expected_recv=19529 actual_valid_recv=19584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=14848 expected_recv=7940 actual_valid_recv=8064 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=14848 expected_recv=18216 expected_overflow=0 exact_recv_per_rank=[10264, 11254, 11255, 10261, 11251, 11251, 0, 0] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=19584 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=8064 slack_slots=21632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=14848 expected_recv=19526 expected_overflow=0 exact_recv_per_rank=[10264, 11254, 11255, 10261, 11251, 11251, 0, 0] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=14848 expected_recv=18216 actual_valid_recv=18432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=14848 expected_recv=19524 expected_overflow=0 exact_recv_per_rank=[10264, 11254, 11255, 10261, 11251, 11251, 0, 0] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=14848 expected_recv=19526 actual_valid_recv=19712 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=18432 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=14848 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=19712 slack_slots=9984 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=14848 expected_recv=19524 actual_valid_recv=19584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=14848 expected_recv=16218 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=19584 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=14848 expected_recv=15883 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=14848 expected_recv=15883 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=14848 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=14848 expected_recv=16229 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=14848 expected_recv=15900 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=14848 expected_recv=16220 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=14848 expected_recv=16229 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=14848 expected_recv=15900 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=14848 expected_recv=16205 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=14848 expected_recv=16205 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=14848 expected_recv=15883 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=14848 expected_recv=15883 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=14848 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=14848 expected_recv=16545 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=14848 expected_recv=16230 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=14848 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=14848 expected_recv=16230 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=14848 expected_recv=16212 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=14848 expected_recv=16220 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=14848 expected_recv=16212 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=14848 expected_recv=16226 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=14848 expected_recv=16226 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[61, 21, 401, 11287, 6787, 16627, 16538, 13814] avg=8192.0000 max=16627 max_rank=5 min=21 min_rank=1 imbalance_ratio=2.029663 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[61, 21, 401, 11287, 6787, 16627, 16538, 13814] avg=8192.0000 max=16627 max_rank=5 min=21 min_rank=1 imbalance_ratio=2.029663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[61, 21, 401, 11287, 6787, 16627, 16538, 13814] avg=8192.0000 max=16627 max_rank=5 min=21 min_rank=1 imbalance_ratio=2.029663 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[61, 21, 401, 11287, 6787, 16627, 16538, 13814] avg=8192.0000 max=16627 max_rank=5 min=21 min_rank=1 imbalance_ratio=2.029663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[7561, 7611, 5178, 9036, 9038, 9037, 9038, 9037] avg=8192.0000 max=9038 max_rank=4 min=5178 min_rank=2 imbalance_ratio=1.103271 max_load_reduction=7589 max_load_reduction_pct=0.456426 moved_to_replicas=33250 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[61, 21, 401, 11287, 6787, 16627, 16538, 13814] avg=8192.0000 max=16627 max_rank=5 min=21 min_rank=1 imbalance_ratio=2.029663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[7561, 7611, 5178, 9036, 9038, 9037, 9038, 9037] avg=8192.0000 max=9038 max_rank=4 min=5178 min_rank=2 imbalance_ratio=1.103271 max_load_reduction=7589 max_load_reduction_pct=0.456426 moved_to_replicas=33250 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[7561, 7611, 5178, 9036, 9038, 9037, 9038, 9037] avg=8192.0000 max=9038 max_rank=4 min=5178 min_rank=2 imbalance_ratio=1.103271 max_load_reduction=7589 max_load_reduction_pct=0.456426 moved_to_replicas=33250 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[61, 21, 401, 11287, 6787, 16627, 16538, 13814] avg=8192.0000 max=16627 max_rank=5 min=21 min_rank=1 imbalance_ratio=2.029663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[61, 21, 401, 11287, 6787, 16627, 16538, 13814] avg=8192.0000 max=16627 max_rank=5 min=21 min_rank=1 imbalance_ratio=2.029663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[7561, 7611, 5178, 9036, 9038, 9037, 9038, 9037] avg=8192.0000 max=9038 max_rank=4 min=5178 min_rank=2 imbalance_ratio=1.103271 max_load_reduction=7589 max_load_reduction_pct=0.456426 moved_to_replicas=33250 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[61, 21, 401, 11287, 6787, 16627, 16538, 13814] avg=8192.0000 max=16627 max_rank=5 min=21 min_rank=1 imbalance_ratio=2.029663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[7561, 7611, 5178, 9036, 9038, 9037, 9038, 9037] avg=8192.0000 max=9038 max_rank=4 min=5178 min_rank=2 imbalance_ratio=1.103271 max_load_reduction=7589 max_load_reduction_pct=0.456426 moved_to_replicas=33250 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[27169, 5, 7, 2, 21548, 11079, 1627, 4099] avg=8192.0000 max=27169 max_rank=0 min=2 min_rank=3 imbalance_ratio=3.316528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[7561, 7611, 5178, 9036, 9038, 9037, 9038, 9037] avg=8192.0000 max=9038 max_rank=4 min=5178 min_rank=2 imbalance_ratio=1.103271 max_load_reduction=7589 max_load_reduction_pct=0.456426 moved_to_replicas=33250 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[7561, 7611, 5178, 9036, 9038, 9037, 9038, 9037] avg=8192.0000 max=9038 max_rank=4 min=5178 min_rank=2 imbalance_ratio=1.103271 max_load_reduction=7589 max_load_reduction_pct=0.456426 moved_to_replicas=33250 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[27169, 5, 7, 2, 21548, 11079, 1627, 4099] avg=8192.0000 max=27169 max_rank=0 min=2 min_rank=3 imbalance_ratio=3.316528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[27169, 5, 7, 2, 21548, 11079, 1627, 4099] avg=8192.0000 max=27169 max_rank=0 min=2 min_rank=3 imbalance_ratio=3.316528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[7561, 7611, 5178, 9036, 9038, 9037, 9038, 9037] avg=8192.0000 max=9038 max_rank=4 min=5178 min_rank=2 imbalance_ratio=1.103271 max_load_reduction=7589 max_load_reduction_pct=0.456426 moved_to_replicas=33250 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[27169, 5, 7, 2, 21548, 11079, 1627, 4099] avg=8192.0000 max=27169 max_rank=0 min=2 min_rank=3 imbalance_ratio=3.316528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[9060, 9058, 9059, 9058, 9060, 9059, 5063, 6119] avg=8192.0000 max=9060 max_rank=0 min=5063 min_rank=6 imbalance_ratio=1.105957 max_load_reduction=18109 max_load_reduction_pct=0.666532 moved_to_replicas=60820 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[27169, 5, 7, 2, 21548, 11079, 1627, 4099] avg=8192.0000 max=27169 max_rank=0 min=2 min_rank=3 imbalance_ratio=3.316528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[9060, 9058, 9059, 9058, 9060, 9059, 5063, 6119] avg=8192.0000 max=9060 max_rank=0 min=5063 min_rank=6 imbalance_ratio=1.105957 max_load_reduction=18109 max_load_reduction_pct=0.666532 moved_to_replicas=60820 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[9060, 9058, 9059, 9058, 9060, 9059, 5063, 6119] avg=8192.0000 max=9060 max_rank=0 min=5063 min_rank=6 imbalance_ratio=1.105957 max_load_reduction=18109 max_load_reduction_pct=0.666532 moved_to_replicas=60820 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[27169, 5, 7, 2, 21548, 11079, 1627, 4099] avg=8192.0000 max=27169 max_rank=0 min=2 min_rank=3 imbalance_ratio=3.316528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[27169, 5, 7, 2, 21548, 11079, 1627, 4099] avg=8192.0000 max=27169 max_rank=0 min=2 min_rank=3 imbalance_ratio=3.316528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[9060, 9058, 9059, 9058, 9060, 9059, 5063, 6119] avg=8192.0000 max=9060 max_rank=0 min=5063 min_rank=6 imbalance_ratio=1.105957 max_load_reduction=18109 max_load_reduction_pct=0.666532 moved_to_replicas=60820 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[9060, 9058, 9059, 9058, 9060, 9059, 5063, 6119] avg=8192.0000 max=9060 max_rank=0 min=5063 min_rank=6 imbalance_ratio=1.105957 max_load_reduction=18109 max_load_reduction_pct=0.666532 moved_to_replicas=60820 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[27169, 5, 7, 2, 21548, 11079, 1627, 4099] avg=8192.0000 max=27169 max_rank=0 min=2 min_rank=3 imbalance_ratio=3.316528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[0, 262, 18, 32479, 6, 5, 32766, 0] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[9060, 9058, 9059, 9058, 9060, 9059, 5063, 6119] avg=8192.0000 max=9060 max_rank=0 min=5063 min_rank=6 imbalance_ratio=1.105957 max_load_reduction=18109 max_load_reduction_pct=0.666532 moved_to_replicas=60820 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[9060, 9058, 9059, 9058, 9060, 9059, 5063, 6119] avg=8192.0000 max=9060 max_rank=0 min=5063 min_rank=6 imbalance_ratio=1.105957 max_load_reduction=18109 max_load_reduction_pct=0.666532 moved_to_replicas=60820 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[0, 262, 18, 32479, 6, 5, 32766, 0] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[0, 262, 18, 32479, 6, 5, 32766, 0] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[9060, 9058, 9059, 9058, 9060, 9059, 5063, 6119] avg=8192.0000 max=9060 max_rank=0 min=5063 min_rank=6 imbalance_ratio=1.105957 max_load_reduction=18109 max_load_reduction_pct=0.666532 moved_to_replicas=60820 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[0, 262, 18, 32479, 6, 5, 32766, 0] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[8275, 7943, 8274, 8274, 8274, 7949, 8275, 8272] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97467 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[0, 262, 18, 32479, 6, 5, 32766, 0] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[8275, 7943, 8274, 8274, 8274, 7949, 8275, 8272] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97467 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[8275, 7943, 8274, 8274, 8274, 7949, 8275, 8272] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97467 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[0, 262, 18, 32479, 6, 5, 32766, 0] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[0, 262, 18, 32479, 6, 5, 32766, 0] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[8275, 7943, 8274, 8274, 8274, 7949, 8275, 8272] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97467 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[8275, 7943, 8274, 8274, 8274, 7949, 8275, 8272] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97467 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[0, 262, 18, 32479, 6, 5, 32766, 0] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[32754, 0, 0, 3, 3, 0, 9, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[8275, 7943, 8274, 8274, 8274, 7949, 8275, 8272] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97467 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[8275, 7943, 8274, 8274, 8274, 7949, 8275, 8272] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97467 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[32754, 0, 0, 3, 3, 0, 9, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[32754, 0, 0, 3, 3, 0, 9, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[32754, 0, 0, 3, 3, 0, 9, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[8275, 7943, 8274, 8274, 8274, 7949, 8275, 8272] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97467 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[8274, 8276, 8275, 8274, 8274, 7944, 7947, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[8274, 8276, 8275, 8274, 8274, 7944, 7947, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[32754, 0, 0, 3, 3, 0, 9, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[8274, 8276, 8275, 8274, 8274, 7944, 7947, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[32754, 0, 0, 3, 3, 0, 9, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[32754, 0, 0, 3, 3, 0, 9, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[8274, 8276, 8275, 8274, 8274, 7944, 7947, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[32766, 1, 1, 1, 0, 0, 1, 32766] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[32754, 0, 0, 3, 3, 0, 9, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[8274, 8276, 8275, 8274, 8274, 7944, 7947, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[8274, 8276, 8275, 8274, 8274, 7944, 7947, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[8274, 8276, 8275, 8274, 8274, 7944, 7947, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[32766, 1, 1, 1, 0, 0, 1, 32766] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[32766, 1, 1, 1, 0, 0, 1, 32766] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[8276, 7945, 8273, 8273, 8274, 8272, 7948, 8275] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[8274, 8276, 8275, 8274, 8274, 7944, 7947, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97935 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[32766, 1, 1, 1, 0, 0, 1, 32766] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[8276, 7945, 8273, 8273, 8274, 8272, 7948, 8275] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[8276, 7945, 8273, 8273, 8274, 8272, 7948, 8275] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[32766, 1, 1, 1, 0, 0, 1, 32766] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[32766, 1, 1, 1, 0, 0, 1, 32766] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[32766, 1, 1, 1, 0, 0, 1, 32766] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[8276, 7945, 8273, 8273, 8274, 8272, 7948, 8275] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[40, 0, 12, 5, 0, 32715, 32764, 0] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[32766, 1, 1, 1, 0, 0, 1, 32766] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[40, 0, 12, 5, 0, 32715, 32764, 0] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[8276, 7945, 8273, 8273, 8274, 8272, 7948, 8275] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[8276, 7945, 8273, 8273, 8274, 8272, 7948, 8275] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[8276, 7945, 8273, 8273, 8274, 8272, 7948, 8275] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[40, 0, 12, 5, 0, 32715, 32764, 0] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[7949, 8275, 8274, 8275, 8275, 8274, 8274, 7940] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97848 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[8276, 7945, 8273, 8273, 8274, 8272, 7948, 8275] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[7949, 8275, 8274, 8275, 8275, 8274, 8274, 7940] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97848 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[40, 0, 12, 5, 0, 32715, 32764, 0] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[7949, 8275, 8274, 8275, 8275, 8274, 8274, 7940] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97848 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[40, 0, 12, 5, 0, 32715, 32764, 0] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[40, 0, 12, 5, 0, 32715, 32764, 0] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[40, 0, 12, 5, 0, 32715, 32764, 0] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[7949, 8275, 8274, 8275, 8275, 8274, 8274, 7940] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97848 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[2069, 7, 0, 0, 1, 30693, 32766, 0] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[2069, 7, 0, 0, 1, 30693, 32766, 0] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[40, 0, 12, 5, 0, 32715, 32764, 0] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[7949, 8275, 8274, 8275, 8275, 8274, 8274, 7940] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97848 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[7949, 8275, 8274, 8275, 8275, 8274, 8274, 7940] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97848 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[7949, 8275, 8274, 8275, 8275, 8274, 8274, 7940] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97848 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[2069, 7, 0, 0, 1, 30693, 32766, 0] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[7949, 8274, 8276, 8276, 8273, 8274, 8272, 7942] avg=8192.0000 max=8276 max_rank=2 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=93087 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[7949, 8274, 8276, 8276, 8273, 8274, 8272, 7942] avg=8192.0000 max=8276 max_rank=2 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=93087 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[7949, 8275, 8274, 8275, 8275, 8274, 8274, 7940] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97848 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[2069, 7, 0, 0, 1, 30693, 32766, 0] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[7949, 8274, 8276, 8276, 8273, 8274, 8272, 7942] avg=8192.0000 max=8276 max_rank=2 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=93087 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[2069, 7, 0, 0, 1, 30693, 32766, 0] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[2069, 7, 0, 0, 1, 30693, 32766, 0] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[2069, 7, 0, 0, 1, 30693, 32766, 0] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[7949, 8274, 8276, 8276, 8273, 8274, 8272, 7942] avg=8192.0000 max=8276 max_rank=2 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=93087 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[0, 32707, 0, 0, 0, 61, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[0, 32707, 0, 0, 0, 61, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[2069, 7, 0, 0, 1, 30693, 32766, 0] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[7949, 8274, 8276, 8276, 8273, 8274, 8272, 7942] avg=8192.0000 max=8276 max_rank=2 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=93087 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[7949, 8274, 8276, 8276, 8273, 8274, 8272, 7942] avg=8192.0000 max=8276 max_rank=2 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=93087 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[7949, 8274, 8276, 8276, 8273, 8274, 8272, 7942] avg=8192.0000 max=8276 max_rank=2 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=93087 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[0, 32707, 0, 0, 0, 61, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[8276, 8275, 8276, 7944, 8274, 7946, 8273, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97872 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[8276, 8275, 8276, 7944, 8274, 7946, 8273, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97872 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[7949, 8274, 8276, 8276, 8273, 8274, 8272, 7942] avg=8192.0000 max=8276 max_rank=2 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=93087 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[0, 32707, 0, 0, 0, 61, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[8276, 8275, 8276, 7944, 8274, 7946, 8273, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97872 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[0, 32707, 0, 0, 0, 61, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[0, 32707, 0, 0, 0, 61, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[0, 32707, 0, 0, 0, 61, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[32767, 1, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[8276, 8275, 8276, 7944, 8274, 7946, 8273, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97872 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[32767, 1, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[0, 32707, 0, 0, 0, 61, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[8276, 8275, 8276, 7944, 8274, 7946, 8273, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97872 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[8276, 8275, 8276, 7944, 8274, 7946, 8273, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97872 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[32767, 1, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[8276, 8275, 8276, 7944, 8274, 7946, 8273, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97872 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[8276, 8275, 8276, 7944, 8274, 7946, 8273, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97872 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[32767, 1, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[32767, 1, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[32767, 1, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[0, 63, 32768, 0, 0, 1, 32704, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[32767, 1, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[0, 63, 32768, 0, 0, 1, 32704, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[32767, 1, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[0, 63, 32768, 0, 0, 1, 32704, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[8276, 7947, 8276, 8272, 7944, 8274, 8273, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97781 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[8276, 7947, 8276, 8272, 7944, 8274, 8273, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97781 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[8276, 7945, 8276, 8276, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[8276, 7947, 8276, 8272, 7944, 8274, 8273, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97781 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[0, 63, 32768, 0, 0, 1, 32704, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[0, 63, 32768, 0, 0, 1, 32704, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[0, 63, 32768, 0, 0, 1, 32704, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[0, 0, 0, 32768, 0, 0, 3, 32765] avg=8192.0000 max=32768 max_rank=3 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[0, 63, 32768, 0, 0, 1, 32704, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[0, 0, 0, 32768, 0, 0, 3, 32765] avg=8192.0000 max=32768 max_rank=3 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[8276, 7947, 8276, 8272, 7944, 8274, 8273, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97781 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[0, 63, 32768, 0, 0, 1, 32704, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[8276, 7947, 8276, 8272, 7944, 8274, 8273, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97781 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[8276, 7947, 8276, 8272, 7944, 8274, 8273, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97781 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[0, 0, 0, 32768, 0, 0, 3, 32765] avg=8192.0000 max=32768 max_rank=3 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[8276, 8276, 7944, 8272, 8275, 8274, 7947, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[8276, 7947, 8276, 8272, 7944, 8274, 8273, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97781 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[8276, 8276, 7944, 8272, 8275, 8274, 7947, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[8276, 7947, 8276, 8272, 7944, 8274, 8273, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97781 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[8276, 8276, 7944, 8272, 8275, 8274, 7947, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[0, 0, 0, 32768, 0, 0, 3, 32765] avg=8192.0000 max=32768 max_rank=3 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[0, 0, 0, 32768, 0, 0, 3, 32765] avg=8192.0000 max=32768 max_rank=3 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[0, 0, 0, 32768, 0, 0, 3, 32765] avg=8192.0000 max=32768 max_rank=3 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[10264, 22505, 0, 0, 32767, 0, 0, 0] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[10264, 22505, 0, 0, 32767, 0, 0, 0] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[0, 0, 0, 32768, 0, 0, 3, 32765] avg=8192.0000 max=32768 max_rank=3 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[8276, 8276, 7944, 8272, 8275, 8274, 7947, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[0, 0, 0, 32768, 0, 0, 3, 32765] avg=8192.0000 max=32768 max_rank=3 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[8276, 8276, 7944, 8272, 8275, 8274, 7947, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[8276, 8276, 7944, 8272, 8275, 8274, 7947, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[10264, 22505, 0, 0, 32767, 0, 0, 0] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[10264, 11254, 11255, 10261, 11251, 11251, 0, 0] avg=8192.0000 max=11255 max_rank=2 min=0 min_rank=6 imbalance_ratio=1.373901 max_load_reduction=21512 max_load_reduction_pct=0.656514 moved_to_replicas=74697 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[10264, 11254, 11255, 10261, 11251, 11251, 0, 0] avg=8192.0000 max=11255 max_rank=2 min=0 min_rank=6 imbalance_ratio=1.373901 max_load_reduction=21512 max_load_reduction_pct=0.656514 moved_to_replicas=74697 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[8276, 8276, 7944, 8272, 8275, 8274, 7947, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[8276, 8276, 7944, 8272, 8275, 8274, 7947, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97954 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[10264, 22505, 0, 0, 32767, 0, 0, 0] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[10264, 11254, 11255, 10261, 11251, 11251, 0, 0] avg=8192.0000 max=11255 max_rank=2 min=0 min_rank=6 imbalance_ratio=1.373901 max_load_reduction=21512 max_load_reduction_pct=0.656514 moved_to_replicas=74697 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[10264, 22505, 0, 0, 32767, 0, 0, 0] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[10264, 22505, 0, 0, 32767, 0, 0, 0] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[0, 0, 32768, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[0, 0, 32768, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[10264, 22505, 0, 0, 32767, 0, 0, 0] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[10264, 22505, 0, 0, 32767, 0, 0, 0] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[10264, 11254, 11255, 10261, 11251, 11251, 0, 0] avg=8192.0000 max=11255 max_rank=2 min=0 min_rank=6 imbalance_ratio=1.373901 max_load_reduction=21512 max_load_reduction_pct=0.656514 moved_to_replicas=74697 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[10264, 11254, 11255, 10261, 11251, 11251, 0, 0] avg=8192.0000 max=11255 max_rank=2 min=0 min_rank=6 imbalance_ratio=1.373901 max_load_reduction=21512 max_load_reduction_pct=0.656514 moved_to_replicas=74697 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[10264, 11254, 11255, 10261, 11251, 11251, 0, 0] avg=8192.0000 max=11255 max_rank=2 min=0 min_rank=6 imbalance_ratio=1.373901 max_load_reduction=21512 max_load_reduction_pct=0.656514 moved_to_replicas=74697 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[0, 0, 32768, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97924 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97924 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[10264, 11254, 11255, 10261, 11251, 11251, 0, 0] avg=8192.0000 max=11255 max_rank=2 min=0 min_rank=6 imbalance_ratio=1.373901 max_load_reduction=21512 max_load_reduction_pct=0.656514 moved_to_replicas=74697 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[10264, 11254, 11255, 10261, 11251, 11251, 0, 0] avg=8192.0000 max=11255 max_rank=2 min=0 min_rank=6 imbalance_ratio=1.373901 max_load_reduction=21512 max_load_reduction_pct=0.656514 moved_to_replicas=74697 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97924 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[0, 0, 32768, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[0, 0, 32768, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[0, 0, 32768, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[0, 0, 1, 0, 32767, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[0, 0, 1, 0, 32767, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97924 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[0, 0, 32768, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[0, 0, 32768, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97924 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[0, 0, 1, 0, 32767, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97924 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97924 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97924 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[0, 0, 1, 0, 32767, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[0, 0, 1, 0, 32767, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[0, 32768, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[0, 0, 1, 0, 32767, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[0, 32768, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[0, 0, 1, 0, 32767, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[0, 0, 1, 0, 32767, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=7 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[0, 32768, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[8276, 8276, 7945, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[0, 32768, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[0, 32768, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[32767, 0, 1, 0, 0, 0, 32768, 0] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[0, 32768, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[32767, 0, 1, 0, 0, 0, 32768, 0] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[0, 32768, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[0, 32768, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[32767, 0, 1, 0, 0, 0, 32768, 0] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[32767, 0, 1, 0, 0, 0, 32768, 0] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[32767, 0, 1, 0, 0, 0, 32768, 0] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[32767, 0, 1, 0, 0, 0, 32768, 0] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[32767, 0, 1, 0, 0, 0, 32768, 0] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[32767, 0, 1, 0, 0, 0, 32768, 0] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 7944, 8272, 8272, 8275] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=32 total_expected_recv=260005 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=211200 peak_demand=11255 required_capacity=1.373901 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=32 total_expected_recv=252384 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=219904 peak_demand=11224 required_capacity=1.370117 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=32 total_expected_recv=267418 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=203520 peak_demand=11224 required_capacity=1.370117 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=32 total_expected_recv=271720 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=200320 peak_demand=11251 required_capacity=1.373413 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=32 total_expected_recv=265574 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=206208 peak_demand=11254 required_capacity=1.373779 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=32 total_expected_recv=247367 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=224768 peak_demand=11224 required_capacity=1.370117 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=32 total_expected_recv=270793 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=201216 peak_demand=11251 required_capacity=1.373413 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=32 total_expected_recv=261891 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=210432 peak_demand=10360 required_capacity=1.264648 + [2026-08-30 05:09:14] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 1233.1 | throughput per GPU (TFLOP/s/GPU): 270.4 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.016278E+01 | loss scale: 1.0 | grad norm: 15.450 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 13312 | min safe capacity: 1.6250 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=14848 expected_recv=17860 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 8467, 8930, 8494, 4396, 8929, 8460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=14848 expected_recv=17487 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 8467, 8930, 8494, 4396, 8929, 8460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=14848 expected_recv=17860 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 8467, 8930, 8494, 4396, 8929, 8460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=14848 expected_recv=8956 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 8467, 8930, 8494, 4396, 8929, 8460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=14848 expected_recv=16235 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 8467, 8930, 8494, 4396, 8929, 8460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=14848 expected_recv=17391 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 8467, 8930, 8494, 4396, 8929, 8460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=14848 expected_recv=17859 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 8467, 8930, 8494, 4396, 8929, 8460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=14848 expected_recv=17860 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=14848 expected_recv=17487 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=14848 expected_recv=17860 actual_valid_recv=18176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=14848 expected_recv=8956 actual_valid_recv=9216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=14848 expected_recv=17424 expected_overflow=0 exact_recv_per_rank=[8930, 8930, 8467, 8930, 8494, 4396, 8929, 8460] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=14848 expected_recv=16235 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17920 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17664 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=18176 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=14848 expected_recv=17391 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=9216 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=14848 expected_recv=17859 actual_valid_recv=17920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=14848 expected_recv=17424 actual_valid_recv=17536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17664 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17920 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17536 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 7942, 8274, 8274, 7949, 8275, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 7942, 8274, 8274, 7949, 8275, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=14848 expected_recv=15880 expected_overflow=0 exact_recv_per_rank=[8274, 7942, 8274, 8274, 7949, 8275, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 7942, 8274, 8274, 7949, 8275, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=14848 expected_recv=15880 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 7942, 8274, 8274, 7949, 8275, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 7942, 8274, 8274, 7949, 8275, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 7942, 8274, 8274, 7949, 8275, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=14848 expected_recv=15904 expected_overflow=0 exact_recv_per_rank=[8274, 7942, 8274, 8274, 7949, 8275, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=14848 expected_recv=15904 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=14848 expected_recv=20404 expected_overflow=0 exact_recv_per_rank=[2083, 10225, 2109, 10224, 10223, 10224, 10224, 10224] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=14848 expected_recv=4303 expected_overflow=0 exact_recv_per_rank=[2083, 10225, 2109, 10224, 10223, 10224, 10224, 10224] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=14848 expected_recv=20405 expected_overflow=0 exact_recv_per_rank=[2083, 10225, 2109, 10224, 10223, 10224, 10224, 10224] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=14848 expected_recv=20404 actual_valid_recv=20480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=14848 expected_recv=4303 actual_valid_recv=4736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=14848 expected_recv=20403 expected_overflow=0 exact_recv_per_rank=[2083, 10225, 2109, 10224, 10223, 10224, 10224, 10224] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=4736 slack_slots=24960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=14848 expected_recv=20405 actual_valid_recv=20480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=20480 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=14848 expected_recv=20403 actual_valid_recv=20480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=20480 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=14848 expected_recv=4352 expected_overflow=0 exact_recv_per_rank=[2083, 10225, 2109, 10224, 10223, 10224, 10224, 10224] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=14848 expected_recv=20402 expected_overflow=0 exact_recv_per_rank=[2083, 10225, 2109, 10224, 10223, 10224, 10224, 10224] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=14848 expected_recv=20401 expected_overflow=0 exact_recv_per_rank=[2083, 10225, 2109, 10224, 10223, 10224, 10224, 10224] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=20480 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=14848 expected_recv=20402 expected_overflow=0 exact_recv_per_rank=[2083, 10225, 2109, 10224, 10223, 10224, 10224, 10224] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=14848 expected_recv=4352 actual_valid_recv=4608 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=14848 expected_recv=20402 actual_valid_recv=20480 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=14848 expected_recv=20401 actual_valid_recv=20736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=14848 expected_recv=20402 actual_valid_recv=20736 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7948, 8274, 7945, 8272, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7948, 8274, 7945, 8272, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=4608 slack_slots=25088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=20480 slack_slots=9216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=20736 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=20736 slack_slots=8960 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7948, 8274, 7945, 8272, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=14848 expected_recv=16214 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7948, 8274, 7945, 8272, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=14848 expected_recv=16214 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=14848 expected_recv=15896 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7948, 8274, 7945, 8272, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7948, 8274, 7945, 8272, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7948, 8274, 7945, 8272, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7948, 8274, 7945, 8272, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=14848 expected_recv=15896 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8274, 7948, 8274, 8275, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=14848 expected_recv=15898 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8274, 7948, 8274, 8275, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=14848 expected_recv=15898 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8274, 7948, 8274, 8275, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8274, 7948, 8274, 8275, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8274, 7948, 8274, 8275, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8274, 7948, 8274, 8275, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=14848 expected_recv=16214 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8274, 7948, 8274, 8275, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 7945, 8274, 7948, 8274, 8275, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=14848 expected_recv=16214 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 7952, 8274, 8276, 8272, 8272, 8275, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 7952, 8274, 8276, 8272, 8272, 8275, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=14848 expected_recv=16212 expected_overflow=0 exact_recv_per_rank=[8276, 7952, 8274, 8276, 8272, 8272, 8275, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=14848 expected_recv=16225 expected_overflow=0 exact_recv_per_rank=[8276, 7952, 8274, 8276, 8272, 8272, 8275, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=14848 expected_recv=16212 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=14848 expected_recv=16225 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 7952, 8274, 8276, 8272, 8272, 8275, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=14848 expected_recv=16227 expected_overflow=0 exact_recv_per_rank=[8276, 7952, 8274, 8276, 8272, 8272, 8275, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=14848 expected_recv=16227 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=14848 expected_recv=16213 expected_overflow=0 exact_recv_per_rank=[8276, 7952, 8274, 8276, 8272, 8272, 8275, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 7952, 8274, 8276, 8272, 8272, 8275, 7939] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=14848 expected_recv=16213 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 8272, 8275, 8274, 7940, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 8272, 8275, 8274, 7940, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 8272, 8275, 8274, 7940, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 8272, 8275, 8274, 7940, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=14848 expected_recv=16213 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 8272, 8275, 8274, 7940, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=14848 expected_recv=15889 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 8272, 8275, 8274, 7940, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=14848 expected_recv=16213 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=14848 expected_recv=15889 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=14848 expected_recv=16224 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 8272, 8275, 8274, 7940, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=14848 expected_recv=16224 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 8272, 8275, 8274, 7940, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=14848 expected_recv=16213 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7939, 8274, 8273, 8274, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7939, 8274, 8273, 8274, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7939, 8274, 8273, 8274, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7939, 8274, 8273, 8274, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=14848 expected_recv=16213 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=14848 expected_recv=16224 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7939, 8274, 8273, 8274, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7939, 8274, 8273, 8274, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=14848 expected_recv=16224 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=14848 expected_recv=15894 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7939, 8274, 8273, 8274, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=14848 expected_recv=15894 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7939, 8274, 8273, 8274, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=14848 expected_recv=16214 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=14848 expected_recv=16214 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=14848 expected_recv=16214 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=14848 expected_recv=16214 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=14848 expected_recv=15898 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=14848 expected_recv=15905 expected_overflow=0 exact_recv_per_rank=[7940, 8275, 8272, 7952, 8275, 8274, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7940, 8275, 8272, 7952, 8275, 8274, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=14848 expected_recv=15877 expected_overflow=0 exact_recv_per_rank=[7940, 8275, 8272, 7952, 8275, 8274, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7940, 8275, 8272, 7952, 8275, 8274, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=14848 expected_recv=15898 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=14848 expected_recv=15905 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7940, 8275, 8272, 7952, 8275, 8274, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=14848 expected_recv=15877 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7940, 8275, 8272, 7952, 8275, 8274, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7940, 8275, 8272, 7952, 8275, 8274, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7940, 8275, 8272, 7952, 8275, 8274, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=14848 expected_recv=15880 expected_overflow=0 exact_recv_per_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=14848 expected_recv=16226 expected_overflow=0 exact_recv_per_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=14848 expected_recv=16225 expected_overflow=0 exact_recv_per_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=14848 expected_recv=15880 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=14848 expected_recv=16226 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=14848 expected_recv=16225 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7951, 8273, 8273, 8272, 8275, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7951, 8273, 8273, 8272, 8275, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7951, 8273, 8273, 8272, 8275, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=14848 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7951, 8273, 8273, 8272, 8275, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=14848 expected_recv=16224 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7951, 8273, 8273, 8272, 8275, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=14848 expected_recv=16217 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=14848 expected_recv=16214 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7951, 8273, 8273, 8272, 8275, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=14848 expected_recv=16227 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7951, 8273, 8273, 8272, 8275, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=14848 expected_recv=16224 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=14848 expected_recv=16214 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=14848 expected_recv=16227 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7951, 8273, 8273, 8272, 8275, 7940] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 7941, 8274, 8273, 8274, 8276, 7950, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=14848 expected_recv=16225 expected_overflow=0 exact_recv_per_rank=[8275, 7941, 8274, 8273, 8274, 8276, 7950, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 7941, 8274, 8273, 8274, 8276, 7950, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=14848 expected_recv=15884 expected_overflow=0 exact_recv_per_rank=[8275, 7941, 8274, 8273, 8274, 8276, 7950, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=14848 expected_recv=16225 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 7941, 8274, 8273, 8274, 8276, 7950, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 7941, 8274, 8273, 8274, 8276, 7950, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=14848 expected_recv=15884 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=14848 expected_recv=16224 expected_overflow=0 exact_recv_per_rank=[8275, 7941, 8274, 8273, 8274, 8276, 7950, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=14848 expected_recv=16224 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 7941, 8274, 8273, 8274, 8276, 7950, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7953, 8275, 8276, 8272, 8273, 8274, 7939, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[7953, 8275, 8276, 8272, 8273, 8274, 7939, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=14848 expected_recv=15902 expected_overflow=0 exact_recv_per_rank=[7953, 8275, 8276, 8272, 8273, 8274, 7939, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[7953, 8275, 8276, 8272, 8273, 8274, 7939, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=14848 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[7953, 8275, 8276, 8272, 8273, 8274, 7939, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7953, 8275, 8276, 8272, 8273, 8274, 7939, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=14848 expected_recv=15902 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=14848 expected_recv=16217 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=14848 expected_recv=16212 expected_overflow=0 exact_recv_per_rank=[7953, 8275, 8276, 8272, 8273, 8274, 7939, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=14848 expected_recv=16212 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7953, 8275, 8276, 8272, 8273, 8274, 7939, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7939, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7939, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7939, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7939, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=14848 expected_recv=15903 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7939, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7939, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=14848 expected_recv=15903 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=14848 expected_recv=15879 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7939, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7939, 7952] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=14848 expected_recv=15879 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=14848 expected_recv=16229 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=14848 expected_recv=16229 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=14848 expected_recv=16224 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=14848 expected_recv=16224 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=14848 expected_recv=15878 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=14848 expected_recv=15878 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[1182, 5, 3478, 13919, 8494, 4396, 32730, 1332] avg=8192.0000 max=32730 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.995361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[1182, 5, 3478, 13919, 8494, 4396, 32730, 1332] avg=8192.0000 max=32730 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.995361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[1182, 5, 3478, 13919, 8494, 4396, 32730, 1332] avg=8192.0000 max=32730 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.995361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[1182, 5, 3478, 13919, 8494, 4396, 32730, 1332] avg=8192.0000 max=32730 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.995361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[1182, 5, 3478, 13919, 8494, 4396, 32730, 1332] avg=8192.0000 max=32730 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.995361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[1182, 5, 3478, 13919, 8494, 4396, 32730, 1332] avg=8192.0000 max=32730 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.995361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[8930, 8930, 8467, 8930, 8494, 4396, 8929, 8460] avg=8192.0000 max=8930 max_rank=0 min=4396 min_rank=5 imbalance_ratio=1.090088 max_load_reduction=23800 max_load_reduction_pct=0.727162 moved_to_replicas=58184 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[8930, 8930, 8467, 8930, 8494, 4396, 8929, 8460] avg=8192.0000 max=8930 max_rank=0 min=4396 min_rank=5 imbalance_ratio=1.090088 max_load_reduction=23800 max_load_reduction_pct=0.727162 moved_to_replicas=58184 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[1182, 5, 3478, 13919, 8494, 4396, 32730, 1332] avg=8192.0000 max=32730 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.995361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[1182, 5, 3478, 13919, 8494, 4396, 32730, 1332] avg=8192.0000 max=32730 max_rank=6 min=5 min_rank=1 imbalance_ratio=3.995361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[8930, 8930, 8467, 8930, 8494, 4396, 8929, 8460] avg=8192.0000 max=8930 max_rank=0 min=4396 min_rank=5 imbalance_ratio=1.090088 max_load_reduction=23800 max_load_reduction_pct=0.727162 moved_to_replicas=58184 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[8930, 8930, 8467, 8930, 8494, 4396, 8929, 8460] avg=8192.0000 max=8930 max_rank=0 min=4396 min_rank=5 imbalance_ratio=1.090088 max_load_reduction=23800 max_load_reduction_pct=0.727162 moved_to_replicas=58184 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[8930, 8930, 8467, 8930, 8494, 4396, 8929, 8460] avg=8192.0000 max=8930 max_rank=0 min=4396 min_rank=5 imbalance_ratio=1.090088 max_load_reduction=23800 max_load_reduction_pct=0.727162 moved_to_replicas=58184 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[8930, 8930, 8467, 8930, 8494, 4396, 8929, 8460] avg=8192.0000 max=8930 max_rank=0 min=4396 min_rank=5 imbalance_ratio=1.090088 max_load_reduction=23800 max_load_reduction_pct=0.727162 moved_to_replicas=58184 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[8930, 8930, 8467, 8930, 8494, 4396, 8929, 8460] avg=8192.0000 max=8930 max_rank=0 min=4396 min_rank=5 imbalance_ratio=1.090088 max_load_reduction=23800 max_load_reduction_pct=0.727162 moved_to_replicas=58184 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[8930, 8930, 8467, 8930, 8494, 4396, 8929, 8460] avg=8192.0000 max=8930 max_rank=0 min=4396 min_rank=5 imbalance_ratio=1.090088 max_load_reduction=23800 max_load_reduction_pct=0.727162 moved_to_replicas=58184 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[2, 5, 6, 0, 997, 32758, 31716, 52] avg=8192.0000 max=32758 max_rank=5 min=0 min_rank=3 imbalance_ratio=3.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[2, 5, 6, 0, 997, 32758, 31716, 52] avg=8192.0000 max=32758 max_rank=5 min=0 min_rank=3 imbalance_ratio=3.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[2, 5, 6, 0, 997, 32758, 31716, 52] avg=8192.0000 max=32758 max_rank=5 min=0 min_rank=3 imbalance_ratio=3.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[2, 5, 6, 0, 997, 32758, 31716, 52] avg=8192.0000 max=32758 max_rank=5 min=0 min_rank=3 imbalance_ratio=3.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[2, 5, 6, 0, 997, 32758, 31716, 52] avg=8192.0000 max=32758 max_rank=5 min=0 min_rank=3 imbalance_ratio=3.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[2, 5, 6, 0, 997, 32758, 31716, 52] avg=8192.0000 max=32758 max_rank=5 min=0 min_rank=3 imbalance_ratio=3.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[8274, 7942, 8274, 8274, 7949, 8275, 8275, 8273] avg=8192.0000 max=8275 max_rank=5 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24483 max_load_reduction_pct=0.747390 moved_to_replicas=95870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[8274, 7942, 8274, 8274, 7949, 8275, 8275, 8273] avg=8192.0000 max=8275 max_rank=5 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24483 max_load_reduction_pct=0.747390 moved_to_replicas=95870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[2, 5, 6, 0, 997, 32758, 31716, 52] avg=8192.0000 max=32758 max_rank=5 min=0 min_rank=3 imbalance_ratio=3.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[8274, 7942, 8274, 8274, 7949, 8275, 8275, 8273] avg=8192.0000 max=8275 max_rank=5 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24483 max_load_reduction_pct=0.747390 moved_to_replicas=95870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[2, 5, 6, 0, 997, 32758, 31716, 52] avg=8192.0000 max=32758 max_rank=5 min=0 min_rank=3 imbalance_ratio=3.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[8274, 7942, 8274, 8274, 7949, 8275, 8275, 8273] avg=8192.0000 max=8275 max_rank=5 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24483 max_load_reduction_pct=0.747390 moved_to_replicas=95870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[8274, 7942, 8274, 8274, 7949, 8275, 8275, 8273] avg=8192.0000 max=8275 max_rank=5 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24483 max_load_reduction_pct=0.747390 moved_to_replicas=95870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[8274, 7942, 8274, 8274, 7949, 8275, 8275, 8273] avg=8192.0000 max=8275 max_rank=5 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24483 max_load_reduction_pct=0.747390 moved_to_replicas=95870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[8274, 7942, 8274, 8274, 7949, 8275, 8275, 8273] avg=8192.0000 max=8275 max_rank=5 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24483 max_load_reduction_pct=0.747390 moved_to_replicas=95870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[2, 32752, 113, 20446, 1, 12220, 0, 2] avg=8192.0000 max=32752 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[8274, 7942, 8274, 8274, 7949, 8275, 8275, 8273] avg=8192.0000 max=8275 max_rank=5 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24483 max_load_reduction_pct=0.747390 moved_to_replicas=95870 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[2, 32752, 113, 20446, 1, 12220, 0, 2] avg=8192.0000 max=32752 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[2, 32752, 113, 20446, 1, 12220, 0, 2] avg=8192.0000 max=32752 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[2, 32752, 113, 20446, 1, 12220, 0, 2] avg=8192.0000 max=32752 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[2083, 10225, 2109, 10224, 10223, 10224, 10224, 10224] avg=8192.0000 max=10225 max_rank=1 min=2083 min_rank=0 imbalance_ratio=1.248169 max_load_reduction=22527 max_load_reduction_pct=0.687805 moved_to_replicas=69604 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[2, 32752, 113, 20446, 1, 12220, 0, 2] avg=8192.0000 max=32752 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[2, 32752, 113, 20446, 1, 12220, 0, 2] avg=8192.0000 max=32752 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[2083, 10225, 2109, 10224, 10223, 10224, 10224, 10224] avg=8192.0000 max=10225 max_rank=1 min=2083 min_rank=0 imbalance_ratio=1.248169 max_load_reduction=22527 max_load_reduction_pct=0.687805 moved_to_replicas=69604 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[2, 32752, 113, 20446, 1, 12220, 0, 2] avg=8192.0000 max=32752 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[2083, 10225, 2109, 10224, 10223, 10224, 10224, 10224] avg=8192.0000 max=10225 max_rank=1 min=2083 min_rank=0 imbalance_ratio=1.248169 max_load_reduction=22527 max_load_reduction_pct=0.687805 moved_to_replicas=69604 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[2, 32752, 113, 20446, 1, 12220, 0, 2] avg=8192.0000 max=32752 max_rank=1 min=0 min_rank=6 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[2083, 10225, 2109, 10224, 10223, 10224, 10224, 10224] avg=8192.0000 max=10225 max_rank=1 min=2083 min_rank=0 imbalance_ratio=1.248169 max_load_reduction=22527 max_load_reduction_pct=0.687805 moved_to_replicas=69604 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[2083, 10225, 2109, 10224, 10223, 10224, 10224, 10224] avg=8192.0000 max=10225 max_rank=1 min=2083 min_rank=0 imbalance_ratio=1.248169 max_load_reduction=22527 max_load_reduction_pct=0.687805 moved_to_replicas=69604 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[2083, 10225, 2109, 10224, 10223, 10224, 10224, 10224] avg=8192.0000 max=10225 max_rank=1 min=2083 min_rank=0 imbalance_ratio=1.248169 max_load_reduction=22527 max_load_reduction_pct=0.687805 moved_to_replicas=69604 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[2083, 10225, 2109, 10224, 10223, 10224, 10224, 10224] avg=8192.0000 max=10225 max_rank=1 min=2083 min_rank=0 imbalance_ratio=1.248169 max_load_reduction=22527 max_load_reduction_pct=0.687805 moved_to_replicas=69604 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[32763, 32762, 4, 0, 2, 1, 2, 2] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[32763, 32762, 4, 0, 2, 1, 2, 2] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[2083, 10225, 2109, 10224, 10223, 10224, 10224, 10224] avg=8192.0000 max=10225 max_rank=1 min=2083 min_rank=0 imbalance_ratio=1.248169 max_load_reduction=22527 max_load_reduction_pct=0.687805 moved_to_replicas=69604 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[32763, 32762, 4, 0, 2, 1, 2, 2] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[32763, 32762, 4, 0, 2, 1, 2, 2] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[8275, 8274, 7948, 8274, 7945, 8272, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[8275, 8274, 7948, 8274, 7945, 8272, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[32763, 32762, 4, 0, 2, 1, 2, 2] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[32763, 32762, 4, 0, 2, 1, 2, 2] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[32763, 32762, 4, 0, 2, 1, 2, 2] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[8275, 8274, 7948, 8274, 7945, 8272, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[8275, 8274, 7948, 8274, 7945, 8272, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[32763, 32762, 4, 0, 2, 1, 2, 2] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[8275, 8274, 7948, 8274, 7945, 8272, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[8275, 8274, 7948, 8274, 7945, 8272, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[8275, 8274, 7948, 8274, 7945, 8272, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[32764, 1, 32742, 21, 4, 4, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[32764, 1, 32742, 21, 4, 4, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[8275, 8274, 7948, 8274, 7945, 8272, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[32764, 1, 32742, 21, 4, 4, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[32764, 1, 32742, 21, 4, 4, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[8276, 7945, 8274, 7948, 8274, 8275, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[8276, 7945, 8274, 7948, 8274, 8275, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[32764, 1, 32742, 21, 4, 4, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[32764, 1, 32742, 21, 4, 4, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[8276, 7945, 8274, 7948, 8274, 8275, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[32764, 1, 32742, 21, 4, 4, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[8276, 7945, 8274, 7948, 8274, 8275, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[32764, 1, 32742, 21, 4, 4, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=6 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[8276, 7945, 8274, 7948, 8274, 8275, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[8276, 7945, 8274, 7948, 8274, 8275, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[32763, 5, 32763, 0, 0, 1, 4, 0] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[8276, 7945, 8274, 7948, 8274, 8275, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[32763, 5, 32763, 0, 0, 1, 4, 0] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[8276, 7945, 8274, 7948, 8274, 8275, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[32763, 5, 32763, 0, 0, 1, 4, 0] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[32763, 5, 32763, 0, 0, 1, 4, 0] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[8276, 7952, 8274, 8276, 8272, 8272, 8275, 7939] avg=8192.0000 max=8276 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[8276, 7952, 8274, 8276, 8272, 8272, 8275, 7939] avg=8192.0000 max=8276 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[32763, 5, 32763, 0, 0, 1, 4, 0] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[32763, 5, 32763, 0, 0, 1, 4, 0] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[8276, 7952, 8274, 8276, 8272, 8272, 8275, 7939] avg=8192.0000 max=8276 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[32763, 5, 32763, 0, 0, 1, 4, 0] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[8276, 7952, 8274, 8276, 8272, 8272, 8275, 7939] avg=8192.0000 max=8276 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[32763, 5, 32763, 0, 0, 1, 4, 0] avg=8192.0000 max=32763 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[8276, 7952, 8274, 8276, 8272, 8272, 8275, 7939] avg=8192.0000 max=8276 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[32763, 0, 1, 4, 0, 32764, 0, 4] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[8276, 7952, 8274, 8276, 8272, 8272, 8275, 7939] avg=8192.0000 max=8276 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[8276, 7952, 8274, 8276, 8272, 8272, 8275, 7939] avg=8192.0000 max=8276 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[32763, 0, 1, 4, 0, 32764, 0, 4] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[8276, 7952, 8274, 8276, 8272, 8272, 8275, 7939] avg=8192.0000 max=8276 max_rank=0 min=7939 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24487 max_load_reduction_pct=0.747398 moved_to_replicas=97956 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[32763, 0, 1, 4, 0, 32764, 0, 4] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[32763, 0, 1, 4, 0, 32764, 0, 4] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[8276, 8275, 8273, 8272, 8275, 8274, 7940, 7951] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[8276, 8275, 8273, 8272, 8275, 8274, 7940, 7951] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[32763, 0, 1, 4, 0, 32764, 0, 4] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[8276, 8275, 8273, 8272, 8275, 8274, 7940, 7951] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[32763, 0, 1, 4, 0, 32764, 0, 4] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[8276, 8275, 8273, 8272, 8275, 8274, 7940, 7951] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[32763, 0, 1, 4, 0, 32764, 0, 4] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[32763, 0, 1, 4, 0, 32764, 0, 4] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[8276, 8275, 8273, 8272, 8275, 8274, 7940, 7951] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[0, 0, 5, 0, 5, 32761, 32755, 10] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[8276, 8275, 8273, 8272, 8275, 8274, 7940, 7951] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[0, 0, 5, 0, 5, 32761, 32755, 10] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[8276, 8275, 8273, 8272, 8275, 8274, 7940, 7951] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[0, 0, 5, 0, 5, 32761, 32755, 10] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[0, 0, 5, 0, 5, 32761, 32755, 10] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[8276, 8275, 8273, 8272, 8275, 8274, 7940, 7951] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[8275, 8274, 8275, 7939, 8274, 8273, 8274, 7952] avg=8192.0000 max=8275 max_rank=0 min=7939 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[8275, 8274, 8275, 7939, 8274, 8273, 8274, 7952] avg=8192.0000 max=8275 max_rank=0 min=7939 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[0, 0, 5, 0, 5, 32761, 32755, 10] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[8275, 8274, 8275, 7939, 8274, 8273, 8274, 7952] avg=8192.0000 max=8275 max_rank=0 min=7939 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[8275, 8274, 8275, 7939, 8274, 8273, 8274, 7952] avg=8192.0000 max=8275 max_rank=0 min=7939 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[0, 0, 5, 0, 5, 32761, 32755, 10] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[0, 0, 5, 0, 5, 32761, 32755, 10] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[0, 0, 5, 0, 5, 32761, 32755, 10] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[8275, 8274, 8275, 7939, 8274, 8273, 8274, 7952] avg=8192.0000 max=8275 max_rank=0 min=7939 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[0, 32760, 4, 1, 5, 32762, 1, 3] avg=8192.0000 max=32762 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[0, 32760, 4, 1, 5, 32762, 1, 3] avg=8192.0000 max=32762 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[8275, 8274, 8275, 7939, 8274, 8273, 8274, 7952] avg=8192.0000 max=8275 max_rank=0 min=7939 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[8275, 8274, 8275, 7939, 8274, 8273, 8274, 7952] avg=8192.0000 max=8275 max_rank=0 min=7939 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[0, 32760, 4, 1, 5, 32762, 1, 3] avg=8192.0000 max=32762 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[0, 32760, 4, 1, 5, 32762, 1, 3] avg=8192.0000 max=32762 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[8275, 8274, 8275, 7939, 8274, 8273, 8274, 7952] avg=8192.0000 max=8275 max_rank=0 min=7939 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[0, 32760, 4, 1, 5, 32762, 1, 3] avg=8192.0000 max=32762 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[0, 32760, 4, 1, 5, 32762, 1, 3] avg=8192.0000 max=32762 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[0, 32760, 4, 1, 5, 32762, 1, 3] avg=8192.0000 max=32762 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[0, 32760, 4, 1, 5, 32762, 1, 3] avg=8192.0000 max=32762 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[93, 32763, 5, 5, 1, 52, 32575, 42] avg=8192.0000 max=32763 max_rank=1 min=1 min_rank=4 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[93, 32763, 5, 5, 1, 52, 32575, 42] avg=8192.0000 max=32763 max_rank=1 min=1 min_rank=4 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[93, 32763, 5, 5, 1, 52, 32575, 42] avg=8192.0000 max=32763 max_rank=1 min=1 min_rank=4 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[93, 32763, 5, 5, 1, 52, 32575, 42] avg=8192.0000 max=32763 max_rank=1 min=1 min_rank=4 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[8275, 8276, 8272, 8275, 7949, 8274, 7940, 8275] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[7940, 8275, 8272, 7952, 8275, 8274, 8273, 8275] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97564 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[7940, 8275, 8272, 7952, 8275, 8274, 8273, 8275] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97564 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[93, 32763, 5, 5, 1, 52, 32575, 42] avg=8192.0000 max=32763 max_rank=1 min=1 min_rank=4 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[7940, 8275, 8272, 7952, 8275, 8274, 8273, 8275] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97564 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[93, 32763, 5, 5, 1, 52, 32575, 42] avg=8192.0000 max=32763 max_rank=1 min=1 min_rank=4 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[7940, 8275, 8272, 7952, 8275, 8274, 8273, 8275] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97564 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[93, 32763, 5, 5, 1, 52, 32575, 42] avg=8192.0000 max=32763 max_rank=1 min=1 min_rank=4 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[7940, 8275, 8272, 7952, 8275, 8274, 8273, 8275] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97564 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[6, 0, 0, 0, 4, 5, 32759, 32762] avg=8192.0000 max=32762 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[93, 32763, 5, 5, 1, 52, 32575, 42] avg=8192.0000 max=32763 max_rank=1 min=1 min_rank=4 imbalance_ratio=3.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[6, 0, 0, 0, 4, 5, 32759, 32762] avg=8192.0000 max=32762 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[7940, 8275, 8272, 7952, 8275, 8274, 8273, 8275] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97564 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[7940, 8275, 8272, 7952, 8275, 8274, 8273, 8275] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97564 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[6, 0, 0, 0, 4, 5, 32759, 32762] avg=8192.0000 max=32762 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[6, 0, 0, 0, 4, 5, 32759, 32762] avg=8192.0000 max=32762 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[7940, 8275, 8272, 7952, 8275, 8274, 8273, 8275] avg=8192.0000 max=8275 max_rank=1 min=7940 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24488 max_load_reduction_pct=0.747429 moved_to_replicas=97564 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[6, 0, 0, 0, 4, 5, 32759, 32762] avg=8192.0000 max=32762 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[6, 0, 0, 0, 4, 5, 32759, 32762] avg=8192.0000 max=32762 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[6, 0, 0, 0, 4, 5, 32759, 32762] avg=8192.0000 max=32762 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[32762, 0, 6, 32758, 3, 0, 5, 2] avg=8192.0000 max=32762 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[6, 0, 0, 0, 4, 5, 32759, 32762] avg=8192.0000 max=32762 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[32762, 0, 6, 32758, 3, 0, 5, 2] avg=8192.0000 max=32762 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[32762, 0, 6, 32758, 3, 0, 5, 2] avg=8192.0000 max=32762 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[32762, 0, 6, 32758, 3, 0, 5, 2] avg=8192.0000 max=32762 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[8276, 8276, 7951, 8273, 8273, 8272, 8275, 7940] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97941 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[7952, 8276, 8272, 7940, 8274, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=1 min=7940 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[8276, 8276, 7951, 8273, 8273, 8272, 8275, 7940] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97941 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[32762, 0, 6, 32758, 3, 0, 5, 2] avg=8192.0000 max=32762 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[8276, 8276, 7951, 8273, 8273, 8272, 8275, 7940] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97941 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[32762, 0, 6, 32758, 3, 0, 5, 2] avg=8192.0000 max=32762 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[8276, 8276, 7951, 8273, 8273, 8272, 8275, 7940] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97941 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[8276, 8276, 7951, 8273, 8273, 8272, 8275, 7940] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97941 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[32762, 0, 6, 32758, 3, 0, 5, 2] avg=8192.0000 max=32762 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[0, 2, 0, 5, 32762, 4, 6, 32757] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[0, 2, 0, 5, 32762, 4, 6, 32757] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[32762, 0, 6, 32758, 3, 0, 5, 2] avg=8192.0000 max=32762 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[8276, 8276, 7951, 8273, 8273, 8272, 8275, 7940] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97941 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[0, 2, 0, 5, 32762, 4, 6, 32757] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[0, 2, 0, 5, 32762, 4, 6, 32757] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[8276, 8276, 7951, 8273, 8273, 8272, 8275, 7940] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97941 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[8275, 7941, 8274, 8273, 8274, 8276, 7950, 8273] avg=8192.0000 max=8276 max_rank=5 min=7941 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[8275, 7941, 8274, 8273, 8274, 8276, 7950, 8273] avg=8192.0000 max=8276 max_rank=5 min=7941 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[0, 2, 0, 5, 32762, 4, 6, 32757] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[8276, 8276, 7951, 8273, 8273, 8272, 8275, 7940] avg=8192.0000 max=8276 max_rank=0 min=7940 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97941 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[8275, 7941, 8274, 8273, 8274, 8276, 7950, 8273] avg=8192.0000 max=8276 max_rank=5 min=7941 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[8275, 7941, 8274, 8273, 8274, 8276, 7950, 8273] avg=8192.0000 max=8276 max_rank=5 min=7941 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[0, 2, 0, 5, 32762, 4, 6, 32757] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[8275, 7941, 8274, 8273, 8274, 8276, 7950, 8273] avg=8192.0000 max=8276 max_rank=5 min=7941 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[37, 1, 32726, 8, 0, 32759, 1, 4] avg=8192.0000 max=32759 max_rank=5 min=0 min_rank=4 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[0, 2, 0, 5, 32762, 4, 6, 32757] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[37, 1, 32726, 8, 0, 32759, 1, 4] avg=8192.0000 max=32759 max_rank=5 min=0 min_rank=4 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[8275, 7941, 8274, 8273, 8274, 8276, 7950, 8273] avg=8192.0000 max=8276 max_rank=5 min=7941 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[37, 1, 32726, 8, 0, 32759, 1, 4] avg=8192.0000 max=32759 max_rank=5 min=0 min_rank=4 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[0, 2, 0, 5, 32762, 4, 6, 32757] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=0 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[37, 1, 32726, 8, 0, 32759, 1, 4] avg=8192.0000 max=32759 max_rank=5 min=0 min_rank=4 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[7953, 8275, 8276, 8272, 8273, 8274, 7939, 8274] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97886 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[8275, 7941, 8274, 8273, 8274, 8276, 7950, 8273] avg=8192.0000 max=8276 max_rank=5 min=7941 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[7953, 8275, 8276, 8272, 8273, 8274, 7939, 8274] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97886 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[37, 1, 32726, 8, 0, 32759, 1, 4] avg=8192.0000 max=32759 max_rank=5 min=0 min_rank=4 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[7953, 8275, 8276, 8272, 8273, 8274, 7939, 8274] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97886 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[8275, 7941, 8274, 8273, 8274, 8276, 7950, 8273] avg=8192.0000 max=8276 max_rank=5 min=7941 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[7953, 8275, 8276, 8272, 8273, 8274, 7939, 8274] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97886 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[37, 1, 32726, 8, 0, 32759, 1, 4] avg=8192.0000 max=32759 max_rank=5 min=0 min_rank=4 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[7953, 8275, 8276, 8272, 8273, 8274, 7939, 8274] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97886 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[7, 0, 32706, 0, 32762, 6, 0, 55] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[7, 0, 32706, 0, 32762, 6, 0, 55] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[37, 1, 32726, 8, 0, 32759, 1, 4] avg=8192.0000 max=32759 max_rank=5 min=0 min_rank=4 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[7953, 8275, 8276, 8272, 8273, 8274, 7939, 8274] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97886 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[7, 0, 32706, 0, 32762, 6, 0, 55] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[37, 1, 32726, 8, 0, 32759, 1, 4] avg=8192.0000 max=32759 max_rank=5 min=0 min_rank=4 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[7, 0, 32706, 0, 32762, 6, 0, 55] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7939, 7952] avg=8192.0000 max=8275 max_rank=1 min=7939 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7939, 7952] avg=8192.0000 max=8275 max_rank=1 min=7939 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[7953, 8275, 8276, 8272, 8273, 8274, 7939, 8274] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97886 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7939, 7952] avg=8192.0000 max=8275 max_rank=1 min=7939 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[7, 0, 32706, 0, 32762, 6, 0, 55] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[7953, 8275, 8276, 8272, 8273, 8274, 7939, 8274] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24483 max_load_reduction_pct=0.747367 moved_to_replicas=97886 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7939, 7952] avg=8192.0000 max=8275 max_rank=1 min=7939 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[7, 0, 32706, 0, 32762, 6, 0, 55] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[32760, 0, 32755, 0, 1, 7, 5, 8] avg=8192.0000 max=32760 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7939, 7952] avg=8192.0000 max=8275 max_rank=1 min=7939 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[32760, 0, 32755, 0, 1, 7, 5, 8] avg=8192.0000 max=32760 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7939, 7952] avg=8192.0000 max=8275 max_rank=1 min=7939 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[7, 0, 32706, 0, 32762, 6, 0, 55] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[32760, 0, 32755, 0, 1, 7, 5, 8] avg=8192.0000 max=32760 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[7, 0, 32706, 0, 32762, 6, 0, 55] avg=8192.0000 max=32762 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[32760, 0, 32755, 0, 1, 7, 5, 8] avg=8192.0000 max=32760 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24484 max_load_reduction_pct=0.747375 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24484 max_load_reduction_pct=0.747375 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[32760, 0, 32755, 0, 1, 7, 5, 8] avg=8192.0000 max=32760 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7939, 7952] avg=8192.0000 max=8275 max_rank=1 min=7939 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24484 max_load_reduction_pct=0.747375 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 8274, 8274, 7939, 7952] avg=8192.0000 max=8275 max_rank=1 min=7939 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24487 max_load_reduction_pct=0.747421 moved_to_replicas=97828 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24484 max_load_reduction_pct=0.747375 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[32760, 0, 32755, 0, 1, 7, 5, 8] avg=8192.0000 max=32760 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24484 max_load_reduction_pct=0.747375 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24484 max_load_reduction_pct=0.747375 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[32760, 0, 32755, 0, 1, 7, 5, 8] avg=8192.0000 max=32760 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[32760, 0, 32755, 0, 1, 7, 5, 8] avg=8192.0000 max=32760 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999023 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24484 max_load_reduction_pct=0.747375 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[8274, 8274, 8276, 8274, 7939, 8275, 8273, 7951] avg=8192.0000 max=8276 max_rank=2 min=7939 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24484 max_load_reduction_pct=0.747375 moved_to_replicas=97938 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=32 total_expected_recv=259395 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=211840 peak_demand=10224 required_capacity=1.248047 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=32 total_expected_recv=266518 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=203648 peak_demand=10224 required_capacity=1.248047 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=32 total_expected_recv=267640 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=204032 peak_demand=10224 required_capacity=1.248047 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=32 total_expected_recv=250290 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=220800 peak_demand=8467 required_capacity=1.033569 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=32 total_expected_recv=251834 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=220416 peak_demand=8930 required_capacity=1.090088 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=32 total_expected_recv=267204 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=203904 peak_demand=10223 required_capacity=1.247925 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=32 total_expected_recv=267305 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=204160 peak_demand=10225 required_capacity=1.248169 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=32 total_expected_recv=266966 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=204160 peak_demand=10224 required_capacity=1.248047 + [2026-08-30 05:09:15] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 932.2 | throughput per GPU (TFLOP/s/GPU): 357.6 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.020984E+01 | loss scale: 1.0 | grad norm: 46.672 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 13312 | min safe capacity: 1.6250 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=14848 expected_recv=16325 expected_overflow=0 exact_recv_per_rank=[8603, 8601, 8162, 8602, 7298, 7065, 8603, 8602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=14848 expected_recv=16876 expected_overflow=0 exact_recv_per_rank=[8603, 8601, 8162, 8602, 7298, 7065, 8603, 8602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=14848 expected_recv=16876 expected_overflow=0 exact_recv_per_rank=[8603, 8601, 8162, 8602, 7298, 7065, 8603, 8602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=14848 expected_recv=16875 expected_overflow=0 exact_recv_per_rank=[8603, 8601, 8162, 8602, 7298, 7065, 8603, 8602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=14848 expected_recv=15108 expected_overflow=0 exact_recv_per_rank=[8603, 8601, 8162, 8602, 7298, 7065, 8603, 8602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=14848 expected_recv=16876 expected_overflow=0 exact_recv_per_rank=[8603, 8601, 8162, 8602, 7298, 7065, 8603, 8602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=14848 expected_recv=16877 expected_overflow=0 exact_recv_per_rank=[8603, 8601, 8162, 8602, 7298, 7065, 8603, 8602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=14848 expected_recv=16325 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=14848 expected_recv=16876 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=14848 expected_recv=16876 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=14848 expected_recv=15259 expected_overflow=0 exact_recv_per_rank=[8603, 8601, 8162, 8602, 7298, 7065, 8603, 8602] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=14848 expected_recv=16875 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=14848 expected_recv=15108 actual_valid_recv=15360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=15360 slack_slots=14336 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=14848 expected_recv=16876 actual_valid_recv=17280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=14848 expected_recv=16877 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17280 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=14848 expected_recv=15259 actual_valid_recv=15488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17280 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=15488 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=14848 expected_recv=16222 expected_overflow=0 exact_recv_per_rank=[7945, 8275, 7948, 8274, 8273, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=14848 expected_recv=16222 expected_overflow=0 exact_recv_per_rank=[7945, 8275, 7948, 8274, 8273, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7945, 8275, 7948, 8274, 8273, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[7945, 8275, 7948, 8274, 8273, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=14848 expected_recv=16222 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7945, 8275, 7948, 8274, 8273, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=14848 expected_recv=16222 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7945, 8275, 7948, 8274, 8273, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7945, 8275, 7948, 8274, 8273, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=14848 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[7945, 8275, 7948, 8274, 8273, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=14848 expected_recv=16217 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 7951, 8275, 8272, 8273, 8273, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=14848 expected_recv=15900 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 7951, 8275, 8272, 8273, 8273, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=14848 expected_recv=15900 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 7951, 8275, 8272, 8273, 8273, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 7951, 8275, 8272, 8273, 8273, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=14848 expected_recv=16215 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 7951, 8275, 8272, 8273, 8273, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 7951, 8275, 8272, 8273, 8273, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 7951, 8275, 8272, 8273, 8273, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=14848 expected_recv=16215 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 7951, 8275, 8272, 8273, 8273, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=14848 expected_recv=16215 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=14848 expected_recv=16215 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 8274, 8273, 7947, 8273, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 8274, 8273, 7947, 8273, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=14848 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 8274, 8273, 7947, 8273, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=14848 expected_recv=15895 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 8274, 8273, 7947, 8273, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=14848 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 8274, 8273, 7947, 8273, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 8274, 8273, 7947, 8273, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 8274, 8273, 7947, 8273, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 8274, 8273, 7947, 8273, 7947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=14848 expected_recv=16220 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=14848 expected_recv=16217 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=14848 expected_recv=15895 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 8275, 8274, 7943, 8274, 7948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 8275, 8274, 7943, 8274, 7948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=14848 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 8275, 8274, 7943, 8274, 7948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 8275, 8274, 7943, 8274, 7948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=14848 expected_recv=16221 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 8275, 8274, 7943, 8274, 7948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=14848 expected_recv=16224 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 8275, 8274, 7943, 8274, 7948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=14848 expected_recv=16217 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=14848 expected_recv=16215 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 8275, 8274, 7943, 8274, 7948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 8275, 8274, 7943, 8274, 7948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=14848 expected_recv=16224 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=14848 expected_recv=16221 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=14848 expected_recv=16215 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 7946, 8272, 8273, 7948, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 7946, 8272, 8273, 7948, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 7946, 8272, 8273, 7948, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=14848 expected_recv=16213 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 7946, 8272, 8273, 7948, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 7946, 8272, 8273, 7948, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 7946, 8272, 8273, 7948, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 7946, 8272, 8273, 7948, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=14848 expected_recv=15898 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 7946, 8272, 8273, 7948, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=14848 expected_recv=16213 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=14848 expected_recv=15898 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=14848 expected_recv=16227 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=14848 expected_recv=16227 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=14848 expected_recv=16215 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=14848 expected_recv=16223 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=14848 expected_recv=16213 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=14848 expected_recv=16215 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=14848 expected_recv=16223 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8274, 8272, 7949, 8275, 8273, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=14848 expected_recv=16213 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=14848 expected_recv=16228 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8274, 8272, 7949, 8275, 8273, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=14848 expected_recv=16228 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8274, 8272, 7949, 8275, 8273, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=14848 expected_recv=16224 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8274, 8272, 7949, 8275, 8273, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=14848 expected_recv=15880 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8274, 8272, 7949, 8275, 8273, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8274, 8272, 7949, 8275, 8273, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8274, 8272, 7949, 8275, 8273, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=14848 expected_recv=16224 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=14848 expected_recv=15880 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8274, 8272, 7949, 8275, 8273, 7942] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=14848 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=14848 expected_recv=15899 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=14848 expected_recv=16214 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=14848 expected_recv=16218 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=14848 expected_recv=15899 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7947, 8272, 8275, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=14848 expected_recv=15902 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7947, 8272, 8275, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=14848 expected_recv=16214 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=14848 expected_recv=15902 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7947, 8272, 8275, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7947, 8272, 8275, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=14848 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7947, 8272, 8275, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=14848 expected_recv=16545 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7947, 8272, 8275, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7947, 8272, 8275, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 7942, 8275, 7948, 8274, 8274, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=14848 expected_recv=15882 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7947, 8272, 8275, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=14848 expected_recv=15902 expected_overflow=0 exact_recv_per_rank=[8274, 7942, 8275, 7948, 8274, 8274, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=14848 expected_recv=15882 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=14848 expected_recv=15902 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 7942, 8275, 7948, 8274, 8274, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 7942, 8275, 7948, 8274, 8274, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 7942, 8275, 7948, 8274, 8274, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 8274, 7946, 8276, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 7942, 8275, 7948, 8274, 8274, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=14848 expected_recv=15881 expected_overflow=0 exact_recv_per_rank=[8274, 7942, 8275, 7948, 8274, 8274, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8274, 7942, 8275, 7948, 8274, 8274, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 8274, 7946, 8276, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=14848 expected_recv=15881 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=14848 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 8274, 7946, 8276, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=14848 expected_recv=16220 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=14848 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 8274, 7946, 8276, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=14848 expected_recv=15896 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 8274, 7946, 8276, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=14848 expected_recv=16218 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=14848 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 7948, 8275, 7943, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=14848 expected_recv=15896 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 8274, 7946, 8276, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=14848 expected_recv=16545 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 8274, 7946, 8276, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=14848 expected_recv=15903 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 7948, 8275, 7943, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8274, 8274, 7946, 8276, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=14848 expected_recv=15903 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 7948, 8275, 7943, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=14848 expected_recv=15880 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 7948, 8275, 7943, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=14848 expected_recv=15880 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 7948, 8275, 7943, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 7948, 8275, 7943, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 7948, 8275, 7943, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8272, 7948, 8275, 7943, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=14848 expected_recv=15903 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=14848 expected_recv=15903 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=14848 expected_recv=15876 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=14848 expected_recv=15876 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=14848 expected_recv=15903 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=14848 expected_recv=15903 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 7942, 8275, 8275, 7948, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=14848 expected_recv=15880 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=14848 expected_recv=15880 actual_valid_recv=16000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=14848 expected_recv=16212 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 7942, 8275, 8275, 7948, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=14848 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 7942, 8275, 8275, 7948, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16000 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=14848 expected_recv=16212 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=14848 expected_recv=16217 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 7942, 8275, 8275, 7948, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 7942, 8275, 8275, 7948, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=14848 expected_recv=15901 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 7942, 8275, 8275, 7948, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[12627, 11, 4138, 13304, 2596, 114, 32743, 3] avg=8192.0000 max=32743 max_rank=6 min=3 min_rank=7 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 7942, 8275, 8275, 7948, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 7942, 8275, 8275, 7948, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=14848 expected_recv=15901 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[12627, 11, 4138, 13304, 2596, 114, 32743, 3] avg=8192.0000 max=32743 max_rank=6 min=3 min_rank=7 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[12627, 11, 4138, 13304, 2596, 114, 32743, 3] avg=8192.0000 max=32743 max_rank=6 min=3 min_rank=7 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[12627, 11, 4138, 13304, 2596, 114, 32743, 3] avg=8192.0000 max=32743 max_rank=6 min=3 min_rank=7 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[8603, 8601, 8162, 8602, 7298, 7065, 8603, 8602] avg=8192.0000 max=8603 max_rank=0 min=7065 min_rank=5 imbalance_ratio=1.050171 max_load_reduction=24140 max_load_reduction_pct=0.737257 moved_to_replicas=65685 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[12627, 11, 4138, 13304, 2596, 114, 32743, 3] avg=8192.0000 max=32743 max_rank=6 min=3 min_rank=7 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[8603, 8601, 8162, 8602, 7298, 7065, 8603, 8602] avg=8192.0000 max=8603 max_rank=0 min=7065 min_rank=5 imbalance_ratio=1.050171 max_load_reduction=24140 max_load_reduction_pct=0.737257 moved_to_replicas=65685 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[8603, 8601, 8162, 8602, 7298, 7065, 8603, 8602] avg=8192.0000 max=8603 max_rank=0 min=7065 min_rank=5 imbalance_ratio=1.050171 max_load_reduction=24140 max_load_reduction_pct=0.737257 moved_to_replicas=65685 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[8603, 8601, 8162, 8602, 7298, 7065, 8603, 8602] avg=8192.0000 max=8603 max_rank=0 min=7065 min_rank=5 imbalance_ratio=1.050171 max_load_reduction=24140 max_load_reduction_pct=0.737257 moved_to_replicas=65685 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[12627, 11, 4138, 13304, 2596, 114, 32743, 3] avg=8192.0000 max=32743 max_rank=6 min=3 min_rank=7 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[4, 2, 44, 4, 9, 32759, 2, 32712] avg=8192.0000 max=32759 max_rank=5 min=2 min_rank=1 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[12627, 11, 4138, 13304, 2596, 114, 32743, 3] avg=8192.0000 max=32743 max_rank=6 min=3 min_rank=7 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[8603, 8601, 8162, 8602, 7298, 7065, 8603, 8602] avg=8192.0000 max=8603 max_rank=0 min=7065 min_rank=5 imbalance_ratio=1.050171 max_load_reduction=24140 max_load_reduction_pct=0.737257 moved_to_replicas=65685 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[12627, 11, 4138, 13304, 2596, 114, 32743, 3] avg=8192.0000 max=32743 max_rank=6 min=3 min_rank=7 imbalance_ratio=3.996948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[4, 2, 44, 4, 9, 32759, 2, 32712] avg=8192.0000 max=32759 max_rank=5 min=2 min_rank=1 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[8603, 8601, 8162, 8602, 7298, 7065, 8603, 8602] avg=8192.0000 max=8603 max_rank=0 min=7065 min_rank=5 imbalance_ratio=1.050171 max_load_reduction=24140 max_load_reduction_pct=0.737257 moved_to_replicas=65685 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[4, 2, 44, 4, 9, 32759, 2, 32712] avg=8192.0000 max=32759 max_rank=5 min=2 min_rank=1 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[4, 2, 44, 4, 9, 32759, 2, 32712] avg=8192.0000 max=32759 max_rank=5 min=2 min_rank=1 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[7945, 8275, 7948, 8274, 8273, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24484 max_load_reduction_pct=0.747398 moved_to_replicas=97852 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[8603, 8601, 8162, 8602, 7298, 7065, 8603, 8602] avg=8192.0000 max=8603 max_rank=0 min=7065 min_rank=5 imbalance_ratio=1.050171 max_load_reduction=24140 max_load_reduction_pct=0.737257 moved_to_replicas=65685 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[8603, 8601, 8162, 8602, 7298, 7065, 8603, 8602] avg=8192.0000 max=8603 max_rank=0 min=7065 min_rank=5 imbalance_ratio=1.050171 max_load_reduction=24140 max_load_reduction_pct=0.737257 moved_to_replicas=65685 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[7945, 8275, 7948, 8274, 8273, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24484 max_load_reduction_pct=0.747398 moved_to_replicas=97852 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[4, 2, 44, 4, 9, 32759, 2, 32712] avg=8192.0000 max=32759 max_rank=5 min=2 min_rank=1 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[7945, 8275, 7948, 8274, 8273, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24484 max_load_reduction_pct=0.747398 moved_to_replicas=97852 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[7945, 8275, 7948, 8274, 8273, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24484 max_load_reduction_pct=0.747398 moved_to_replicas=97852 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[4, 2, 44, 4, 9, 32759, 2, 32712] avg=8192.0000 max=32759 max_rank=5 min=2 min_rank=1 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[2, 32764, 9, 0, 0, 32759, 2, 0] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[7945, 8275, 7948, 8274, 8273, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24484 max_load_reduction_pct=0.747398 moved_to_replicas=97852 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[4, 2, 44, 4, 9, 32759, 2, 32712] avg=8192.0000 max=32759 max_rank=5 min=2 min_rank=1 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[2, 32764, 9, 0, 0, 32759, 2, 0] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[4, 2, 44, 4, 9, 32759, 2, 32712] avg=8192.0000 max=32759 max_rank=5 min=2 min_rank=1 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[2, 32764, 9, 0, 0, 32759, 2, 0] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[2, 32764, 9, 0, 0, 32759, 2, 0] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[7945, 8275, 7948, 8274, 8273, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24484 max_load_reduction_pct=0.747398 moved_to_replicas=97852 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[8275, 8275, 7951, 8275, 8272, 8273, 8273, 7942] avg=8192.0000 max=8275 max_rank=0 min=7942 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[7945, 8275, 7948, 8274, 8273, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24484 max_load_reduction_pct=0.747398 moved_to_replicas=97852 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[8275, 8275, 7951, 8275, 8272, 8273, 8273, 7942] avg=8192.0000 max=8275 max_rank=0 min=7942 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[7945, 8275, 7948, 8274, 8273, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010132 max_load_reduction=24484 max_load_reduction_pct=0.747398 moved_to_replicas=97852 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[2, 32764, 9, 0, 0, 32759, 2, 0] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[8275, 8275, 7951, 8275, 8272, 8273, 8273, 7942] avg=8192.0000 max=8275 max_rank=0 min=7942 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[8275, 8275, 7951, 8275, 8272, 8273, 8273, 7942] avg=8192.0000 max=8275 max_rank=0 min=7942 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[2, 32765, 1, 32762, 1, 1, 2, 2] avg=8192.0000 max=32765 max_rank=1 min=1 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[2, 32764, 9, 0, 0, 32759, 2, 0] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[2, 32764, 9, 0, 0, 32759, 2, 0] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[2, 32765, 1, 32762, 1, 1, 2, 2] avg=8192.0000 max=32765 max_rank=1 min=1 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[8275, 8275, 7951, 8275, 8272, 8273, 8273, 7942] avg=8192.0000 max=8275 max_rank=0 min=7942 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[2, 32764, 9, 0, 0, 32759, 2, 0] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[2, 32765, 1, 32762, 1, 1, 2, 2] avg=8192.0000 max=32765 max_rank=1 min=1 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[2, 32765, 1, 32762, 1, 1, 2, 2] avg=8192.0000 max=32765 max_rank=1 min=1 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[8274, 8275, 8273, 8274, 8273, 7947, 8273, 7947] avg=8192.0000 max=8275 max_rank=1 min=7947 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[8275, 8275, 7951, 8275, 8272, 8273, 8273, 7942] avg=8192.0000 max=8275 max_rank=0 min=7942 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[8274, 8275, 8273, 8274, 8273, 7947, 8273, 7947] avg=8192.0000 max=8275 max_rank=1 min=7947 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[8275, 8275, 7951, 8275, 8272, 8273, 8273, 7942] avg=8192.0000 max=8275 max_rank=0 min=7942 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[8275, 8275, 7951, 8275, 8272, 8273, 8273, 7942] avg=8192.0000 max=8275 max_rank=0 min=7942 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[8274, 8275, 8273, 8274, 8273, 7947, 8273, 7947] avg=8192.0000 max=8275 max_rank=1 min=7947 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[2, 32765, 1, 32762, 1, 1, 2, 2] avg=8192.0000 max=32765 max_rank=1 min=1 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[8274, 8275, 8273, 8274, 8273, 7947, 8273, 7947] avg=8192.0000 max=8275 max_rank=1 min=7947 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[0, 0, 32765, 32765, 2, 0, 2, 2] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[2, 32765, 1, 32762, 1, 1, 2, 2] avg=8192.0000 max=32765 max_rank=1 min=1 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[0, 0, 32765, 32765, 2, 0, 2, 2] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[8274, 8275, 8273, 8274, 8273, 7947, 8273, 7947] avg=8192.0000 max=8275 max_rank=1 min=7947 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[2, 32765, 1, 32762, 1, 1, 2, 2] avg=8192.0000 max=32765 max_rank=1 min=1 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[2, 32765, 1, 32762, 1, 1, 2, 2] avg=8192.0000 max=32765 max_rank=1 min=1 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[0, 0, 32765, 32765, 2, 0, 2, 2] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[0, 0, 32765, 32765, 2, 0, 2, 2] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[8275, 8275, 8272, 8275, 8274, 7943, 8274, 7948] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[8274, 8275, 8273, 8274, 8273, 7947, 8273, 7947] avg=8192.0000 max=8275 max_rank=1 min=7947 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[8275, 8275, 8272, 8275, 8274, 7943, 8274, 7948] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[8274, 8275, 8273, 8274, 8273, 7947, 8273, 7947] avg=8192.0000 max=8275 max_rank=1 min=7947 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[8274, 8275, 8273, 8274, 8273, 7947, 8273, 7947] avg=8192.0000 max=8275 max_rank=1 min=7947 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[8275, 8275, 8272, 8275, 8274, 7943, 8274, 7948] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[0, 0, 32765, 32765, 2, 0, 2, 2] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[8275, 8275, 8272, 8275, 8274, 7943, 8274, 7948] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[3, 3, 32765, 2, 0, 32760, 3, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[3, 3, 32765, 2, 0, 32760, 3, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[0, 0, 32765, 32765, 2, 0, 2, 2] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[8275, 8275, 8272, 8275, 8274, 7943, 8274, 7948] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[0, 0, 32765, 32765, 2, 0, 2, 2] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[0, 0, 32765, 32765, 2, 0, 2, 2] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[3, 3, 32765, 2, 0, 32760, 3, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[3, 3, 32765, 2, 0, 32760, 3, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[8274, 8274, 8275, 7946, 8272, 8273, 7948, 8274] avg=8192.0000 max=8275 max_rank=2 min=7946 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[8274, 8274, 8275, 7946, 8272, 8273, 7948, 8274] avg=8192.0000 max=8275 max_rank=2 min=7946 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[8275, 8275, 8272, 8275, 8274, 7943, 8274, 7948] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[8275, 8275, 8272, 8275, 8274, 7943, 8274, 7948] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[8275, 8275, 8272, 8275, 8274, 7943, 8274, 7948] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97957 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[3, 3, 32765, 2, 0, 32760, 3, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[8274, 8274, 8275, 7946, 8272, 8273, 7948, 8274] avg=8192.0000 max=8275 max_rank=2 min=7946 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[8274, 8274, 8275, 7946, 8272, 8273, 7948, 8274] avg=8192.0000 max=8275 max_rank=2 min=7946 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[32765, 0, 3, 0, 32765, 0, 0, 3] avg=8192.0000 max=32765 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[32765, 0, 3, 0, 32765, 0, 0, 3] avg=8192.0000 max=32765 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[3, 3, 32765, 2, 0, 32760, 3, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[8274, 8274, 8275, 7946, 8272, 8273, 7948, 8274] avg=8192.0000 max=8275 max_rank=2 min=7946 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[3, 3, 32765, 2, 0, 32760, 3, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[3, 3, 32765, 2, 0, 32760, 3, 0] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[32765, 0, 3, 0, 32765, 0, 0, 3] avg=8192.0000 max=32765 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[32765, 0, 3, 0, 32765, 0, 0, 3] avg=8192.0000 max=32765 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[8274, 8274, 8275, 7946, 8272, 8273, 7948, 8274] avg=8192.0000 max=8275 max_rank=2 min=7946 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[8274, 8274, 8275, 7946, 8272, 8273, 7948, 8274] avg=8192.0000 max=8275 max_rank=2 min=7946 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[8274, 8274, 8275, 7946, 8272, 8273, 7948, 8274] avg=8192.0000 max=8275 max_rank=2 min=7946 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[32765, 0, 3, 0, 32765, 0, 0, 3] avg=8192.0000 max=32765 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[32764, 0, 3, 32765, 3, 0, 1, 0] avg=8192.0000 max=32765 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[32764, 0, 3, 32765, 3, 0, 1, 0] avg=8192.0000 max=32765 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[32765, 0, 3, 0, 32765, 0, 0, 3] avg=8192.0000 max=32765 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[32765, 0, 3, 0, 32765, 0, 0, 3] avg=8192.0000 max=32765 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[32765, 0, 3, 0, 32765, 0, 0, 3] avg=8192.0000 max=32765 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[32764, 0, 3, 32765, 3, 0, 1, 0] avg=8192.0000 max=32765 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[8275, 8276, 8274, 8272, 7949, 8275, 8273, 7942] avg=8192.0000 max=8276 max_rank=1 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[32764, 0, 3, 32765, 3, 0, 1, 0] avg=8192.0000 max=32765 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[8275, 8276, 8274, 8272, 7949, 8275, 8273, 7942] avg=8192.0000 max=8276 max_rank=1 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 8275, 7943, 8272, 7949] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[8275, 8276, 8274, 8272, 7949, 8275, 8273, 7942] avg=8192.0000 max=8276 max_rank=1 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[32764, 0, 3, 32765, 3, 0, 1, 0] avg=8192.0000 max=32765 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[8275, 8276, 8274, 8272, 7949, 8275, 8273, 7942] avg=8192.0000 max=8276 max_rank=1 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[3, 32765, 0, 0, 3, 0, 0, 32765] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[3, 32765, 0, 0, 3, 0, 0, 32765] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[32764, 0, 3, 32765, 3, 0, 1, 0] avg=8192.0000 max=32765 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[8275, 8276, 8274, 8272, 7949, 8275, 8273, 7942] avg=8192.0000 max=8276 max_rank=1 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[32764, 0, 3, 32765, 3, 0, 1, 0] avg=8192.0000 max=32765 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[32764, 0, 3, 32765, 3, 0, 1, 0] avg=8192.0000 max=32765 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[3, 32765, 0, 0, 3, 0, 0, 32765] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[3, 32765, 0, 0, 3, 0, 0, 32765] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[8275, 8276, 8274, 8272, 7949, 8275, 8273, 7942] avg=8192.0000 max=8276 max_rank=1 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[8275, 8276, 8274, 8272, 7949, 8275, 8273, 7942] avg=8192.0000 max=8276 max_rank=1 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[8275, 8276, 8274, 8272, 7949, 8275, 8273, 7942] avg=8192.0000 max=8276 max_rank=1 min=7942 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24489 max_load_reduction_pct=0.747413 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[3, 32765, 0, 0, 3, 0, 0, 32765] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[0, 32765, 3, 3, 0, 32765, 0, 0] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[0, 32765, 3, 3, 0, 32765, 0, 0] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[3, 32765, 0, 0, 3, 0, 0, 32765] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[8275, 8274, 8275, 7947, 8272, 8275, 7944, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[3, 32765, 0, 0, 3, 0, 0, 32765] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[0, 32765, 3, 3, 0, 32765, 0, 0] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[0, 32765, 3, 3, 0, 32765, 0, 0] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[3, 32765, 0, 0, 3, 0, 0, 32765] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[8275, 8274, 8275, 7947, 8272, 8275, 7944, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[8275, 8274, 8275, 7947, 8272, 8275, 7944, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[8275, 8274, 8275, 7947, 8272, 8275, 7944, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[0, 32765, 3, 3, 0, 32765, 0, 0] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[8275, 8275, 8275, 8272, 7947, 7943, 8275, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[3, 312, 0, 4, 32340, 101, 32764, 12] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[3, 312, 0, 4, 32340, 101, 32764, 12] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[0, 32765, 3, 3, 0, 32765, 0, 0] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[8275, 8274, 8275, 7947, 8272, 8275, 7944, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[8274, 7942, 8275, 7948, 8274, 8274, 8274, 8275] avg=8192.0000 max=8275 max_rank=2 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[0, 32765, 3, 3, 0, 32765, 0, 0] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[3, 312, 0, 4, 32340, 101, 32764, 12] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[3, 312, 0, 4, 32340, 101, 32764, 12] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[0, 32765, 3, 3, 0, 32765, 0, 0] avg=8192.0000 max=32765 max_rank=1 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[8274, 7942, 8275, 7948, 8274, 8274, 8274, 8275] avg=8192.0000 max=8275 max_rank=2 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[8275, 8274, 8275, 7947, 8272, 8275, 7944, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[8274, 7942, 8275, 7948, 8274, 8274, 8274, 8275] avg=8192.0000 max=8275 max_rank=2 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[8275, 8274, 8275, 7947, 8272, 8275, 7944, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[8274, 7942, 8275, 7948, 8274, 8274, 8274, 8275] avg=8192.0000 max=8275 max_rank=2 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[3, 312, 0, 4, 32340, 101, 32764, 12] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[32679, 0, 3, 32761, 4, 22, 21, 46] avg=8192.0000 max=32761 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[8275, 8274, 8275, 7947, 8272, 8275, 7944, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97953 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[32679, 0, 3, 32761, 4, 22, 21, 46] avg=8192.0000 max=32761 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[3, 312, 0, 4, 32340, 101, 32764, 12] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[8274, 7942, 8275, 7948, 8274, 8274, 8274, 8275] avg=8192.0000 max=8275 max_rank=2 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[8273, 8274, 8274, 8274, 7946, 8276, 8273, 7946] avg=8192.0000 max=8276 max_rank=5 min=7946 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24485 max_load_reduction_pct=0.747383 moved_to_replicas=97642 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[32679, 0, 3, 32761, 4, 22, 21, 46] avg=8192.0000 max=32761 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[3, 312, 0, 4, 32340, 101, 32764, 12] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[32679, 0, 3, 32761, 4, 22, 21, 46] avg=8192.0000 max=32761 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[8273, 8274, 8274, 8274, 7946, 8276, 8273, 7946] avg=8192.0000 max=8276 max_rank=5 min=7946 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24485 max_load_reduction_pct=0.747383 moved_to_replicas=97642 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[8274, 7942, 8275, 7948, 8274, 8274, 8274, 8275] avg=8192.0000 max=8275 max_rank=2 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[3, 312, 0, 4, 32340, 101, 32764, 12] avg=8192.0000 max=32764 max_rank=6 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[8273, 8274, 8274, 8274, 7946, 8276, 8273, 7946] avg=8192.0000 max=8276 max_rank=5 min=7946 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24485 max_load_reduction_pct=0.747383 moved_to_replicas=97642 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[8274, 7942, 8275, 7948, 8274, 8274, 8274, 8275] avg=8192.0000 max=8275 max_rank=2 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[8273, 8274, 8274, 8274, 7946, 8276, 8273, 7946] avg=8192.0000 max=8276 max_rank=5 min=7946 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24485 max_load_reduction_pct=0.747383 moved_to_replicas=97642 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[32679, 0, 3, 32761, 4, 22, 21, 46] avg=8192.0000 max=32761 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[0, 0, 32765, 4, 32764, 0, 2, 1] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[8274, 7942, 8275, 7948, 8274, 8274, 8274, 8275] avg=8192.0000 max=8275 max_rank=2 min=7942 min_rank=1 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97024 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[0, 0, 32765, 4, 32764, 0, 2, 1] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[32679, 0, 3, 32761, 4, 22, 21, 46] avg=8192.0000 max=32761 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[8273, 8274, 8274, 8274, 7946, 8276, 8273, 7946] avg=8192.0000 max=8276 max_rank=5 min=7946 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24485 max_load_reduction_pct=0.747383 moved_to_replicas=97642 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[0, 0, 32765, 4, 32764, 0, 2, 1] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[8275, 8275, 8272, 7948, 8275, 7943, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[0, 0, 32765, 4, 32764, 0, 2, 1] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[8275, 8275, 8272, 7948, 8275, 7943, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[32679, 0, 3, 32761, 4, 22, 21, 46] avg=8192.0000 max=32761 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[8273, 8274, 8274, 8274, 7946, 8276, 8273, 7946] avg=8192.0000 max=8276 max_rank=5 min=7946 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24485 max_load_reduction_pct=0.747383 moved_to_replicas=97642 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[32679, 0, 3, 32761, 4, 22, 21, 46] avg=8192.0000 max=32761 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[8275, 8275, 8272, 7948, 8275, 7943, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[8275, 8275, 8272, 7948, 8275, 7943, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[0, 0, 32765, 4, 32764, 0, 2, 1] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[8273, 8274, 8274, 8274, 7946, 8276, 8273, 7946] avg=8192.0000 max=8276 max_rank=5 min=7946 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24485 max_load_reduction_pct=0.747383 moved_to_replicas=97642 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[32764, 0, 0, 4, 0, 32764, 2, 2] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[8273, 8274, 8274, 8274, 7946, 8276, 8273, 7946] avg=8192.0000 max=8276 max_rank=5 min=7946 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24485 max_load_reduction_pct=0.747383 moved_to_replicas=97642 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[32764, 0, 0, 4, 0, 32764, 2, 2] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[0, 0, 32765, 4, 32764, 0, 2, 1] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[8275, 8275, 8272, 7948, 8275, 7943, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[32764, 0, 0, 4, 0, 32764, 2, 2] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[32764, 0, 0, 4, 0, 32764, 2, 2] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[0, 0, 32765, 4, 32764, 0, 2, 1] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[8275, 8275, 8272, 7948, 8275, 7943, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[0, 0, 32765, 4, 32764, 0, 2, 1] avg=8192.0000 max=32765 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[32764, 0, 0, 4, 0, 32764, 2, 2] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[2, 2, 32764, 0, 0, 3, 0, 32765] avg=8192.0000 max=32765 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[8275, 8275, 8272, 7948, 8275, 7943, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[8275, 8275, 8272, 7948, 8275, 7943, 8274, 8274] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[2, 2, 32764, 0, 0, 3, 0, 32765] avg=8192.0000 max=32765 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[32764, 0, 0, 4, 0, 32764, 2, 2] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[2, 2, 32764, 0, 0, 3, 0, 32765] avg=8192.0000 max=32765 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[2, 2, 32764, 0, 0, 3, 0, 32765] avg=8192.0000 max=32765 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[32764, 0, 0, 4, 0, 32764, 2, 2] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[32764, 0, 0, 4, 0, 32764, 2, 2] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[2, 2, 32764, 0, 0, 3, 0, 32765] avg=8192.0000 max=32765 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[2, 32762, 32764, 1, 0, 2, 5, 0] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[8275, 8275, 8274, 7948, 7940, 8275, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97948 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[2, 32762, 32764, 1, 0, 2, 5, 0] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[2, 2, 32764, 0, 0, 3, 0, 32765] avg=8192.0000 max=32765 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[2, 32762, 32764, 1, 0, 2, 5, 0] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[8273, 8275, 8275, 7942, 8275, 8275, 7948, 8273] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[8273, 8275, 8275, 7942, 8275, 8275, 7948, 8273] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[2, 32762, 32764, 1, 0, 2, 5, 0] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[2, 2, 32764, 0, 0, 3, 0, 32765] avg=8192.0000 max=32765 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[8273, 8275, 8275, 7942, 8275, 8275, 7948, 8273] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[2, 2, 32764, 0, 0, 3, 0, 32765] avg=8192.0000 max=32765 max_rank=7 min=0 min_rank=3 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[2, 32762, 32764, 1, 0, 2, 5, 0] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[8273, 8275, 8275, 7942, 8275, 8275, 7948, 8273] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[8274, 8273, 8275, 8275, 8272, 7949, 7944, 8274] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97951 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[2, 32762, 32764, 1, 0, 2, 5, 0] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[8273, 8275, 8275, 7942, 8275, 8275, 7948, 8273] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[8273, 8275, 8275, 7942, 8275, 8275, 7948, 8273] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[2, 32762, 32764, 1, 0, 2, 5, 0] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[2, 32762, 32764, 1, 0, 2, 5, 0] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=4 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[8273, 8275, 8275, 7942, 8275, 8275, 7948, 8273] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[8273, 8275, 8275, 7942, 8275, 8275, 7948, 8273] avg=8192.0000 max=8275 max_rank=1 min=7942 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97939 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=32 total_expected_recv=262929 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=208768 peak_demand=8275 required_capacity=1.010132 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=32 total_expected_recv=262127 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=208512 peak_demand=8602 required_capacity=1.050049 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=32 total_expected_recv=260364 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=211584 peak_demand=8276 required_capacity=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=32 total_expected_recv=261526 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=209280 peak_demand=8602 required_capacity=1.050049 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=32 total_expected_recv=260190 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=211072 peak_demand=8275 required_capacity=1.010132 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=32 total_expected_recv=261461 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=208768 peak_demand=8603 required_capacity=1.050171 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=32 total_expected_recv=264451 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=206976 peak_demand=8603 required_capacity=1.050171 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=32 total_expected_recv=264104 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=207872 peak_demand=8601 required_capacity=1.049927 + [2026-08-30 05:09:16] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 920.2 | throughput per GPU (TFLOP/s/GPU): 362.3 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.927358E+00 | loss scale: 1.0 | grad norm: 8.087 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 13312 | min safe capacity: 1.6250 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=14848 expected_recv=24312 expected_overflow=0 exact_recv_per_rank=[14064, 27, 14064, 13036, 5451, 64, 4766, 14064] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=14848 expected_recv=23287 expected_overflow=0 exact_recv_per_rank=[14064, 27, 14064, 13036, 5451, 64, 4766, 14064] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=14848 expected_recv=5884 expected_overflow=0 exact_recv_per_rank=[14064, 27, 14064, 13036, 5451, 64, 4766, 14064] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=14848 expected_recv=24313 expected_overflow=0 exact_recv_per_rank=[14064, 27, 14064, 13036, 5451, 64, 4766, 14064] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=14848 expected_recv=3319 expected_overflow=0 exact_recv_per_rank=[14064, 27, 14064, 13036, 5451, 64, 4766, 14064] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=14848 expected_recv=19241 expected_overflow=0 exact_recv_per_rank=[14064, 27, 14064, 13036, 5451, 64, 4766, 14064] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=14848 expected_recv=15016 expected_overflow=0 exact_recv_per_rank=[14064, 27, 14064, 13036, 5451, 64, 4766, 14064] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=14848 expected_recv=24312 actual_valid_recv=24448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=14848 expected_recv=23287 actual_valid_recv=23424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=14848 expected_recv=5884 actual_valid_recv=6144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=14848 expected_recv=15700 expected_overflow=0 exact_recv_per_rank=[14064, 27, 14064, 13036, 5451, 64, 4766, 14064] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=14848 expected_recv=24313 actual_valid_recv=24448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=14848 expected_recv=3319 actual_valid_recv=3584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=24448 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=23424 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=14848 expected_recv=19241 actual_valid_recv=19584 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=14848 expected_recv=15016 actual_valid_recv=15488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=6144 slack_slots=23552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=24448 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=3584 slack_slots=26112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=14848 expected_recv=15700 actual_valid_recv=15872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=19584 slack_slots=10112 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=15488 slack_slots=14208 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=15872 slack_slots=13824 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 7950, 8274, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=14848 expected_recv=16223 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 7950, 8274, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 7950, 8274, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 7950, 8274, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 7950, 8274, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=14848 expected_recv=16223 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 7950, 8274, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=14848 expected_recv=16215 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 7950, 8274, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=14848 expected_recv=16223 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 8275, 7950, 8274, 8274, 7940, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=14848 expected_recv=16215 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=14848 expected_recv=16223 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7956, 8275, 8274, 7936, 8272, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=14848 expected_recv=16209 expected_overflow=0 exact_recv_per_rank=[7956, 8275, 8274, 7936, 8272, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7956, 8275, 8274, 7936, 8272, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7956, 8275, 8274, 7936, 8272, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[7956, 8275, 8274, 7936, 8272, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=14848 expected_recv=15901 expected_overflow=0 exact_recv_per_rank=[7956, 8275, 8274, 7936, 8272, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=14848 expected_recv=16221 expected_overflow=0 exact_recv_per_rank=[7956, 8275, 8274, 7936, 8272, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7956, 8275, 8274, 7936, 8272, 8275, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=14848 expected_recv=16209 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=14848 expected_recv=15901 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=14848 expected_recv=16221 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=14848 expected_recv=15887 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 7942, 8274, 8275, 8273, 8275, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=14848 expected_recv=15887 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 7942, 8274, 8275, 8273, 8275, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 7942, 8274, 8275, 8273, 8275, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=14848 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 7942, 8274, 8275, 8273, 8275, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=14848 expected_recv=16224 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 7942, 8274, 8275, 8273, 8275, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 7942, 8274, 8275, 8273, 8275, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=14848 expected_recv=16545 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 7942, 8274, 8275, 8273, 8275, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=14848 expected_recv=16222 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 7942, 8274, 8275, 8273, 8275, 7951] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=14848 expected_recv=16224 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=14848 expected_recv=16222 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8273, 8273, 7953, 8273, 7938, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8273, 8273, 7953, 8273, 7938, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8273, 8273, 7953, 8273, 7938, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8273, 8273, 7953, 8273, 7938, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8273, 8273, 7953, 8273, 7938, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8273, 8273, 7953, 8273, 7938, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=14848 expected_recv=16225 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8273, 8273, 7953, 8273, 7938, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=14848 expected_recv=15886 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8273, 8273, 7953, 8273, 7938, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=14848 expected_recv=16225 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=14848 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 7941, 8275, 7952, 8275, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=14848 expected_recv=15886 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=14848 expected_recv=16217 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 7941, 8275, 7952, 8275, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 7941, 8275, 7952, 8275, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 7941, 8275, 7952, 8275, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 7941, 8275, 7952, 8275, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 7941, 8275, 7952, 8275, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=14848 expected_recv=15899 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 7941, 8275, 7952, 8275, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8274, 8274, 7941, 8275, 7952, 8275, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7955, 8275, 8273, 8273, 8276, 8275, 7935, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=14848 expected_recv=15899 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7955, 8275, 8273, 8273, 8276, 8275, 7935, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7955, 8275, 8273, 8273, 8276, 8275, 7935, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7955, 8275, 8273, 8273, 8276, 8275, 7935, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7955, 8275, 8273, 8273, 8276, 8275, 7935, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=14848 expected_recv=15901 expected_overflow=0 exact_recv_per_rank=[7955, 8275, 8273, 8273, 8276, 8275, 7935, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7955, 8275, 8273, 8273, 8276, 8275, 7935, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=14848 expected_recv=15879 expected_overflow=0 exact_recv_per_rank=[7955, 8275, 8273, 8273, 8276, 8275, 7935, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=14848 expected_recv=15901 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 7933, 8274, 8274, 7959, 8273, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=14848 expected_recv=15879 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 7933, 8274, 8274, 7959, 8273, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=14848 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8276, 7933, 8274, 8274, 7959, 8273, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=14848 expected_recv=16209 expected_overflow=0 exact_recv_per_rank=[8276, 7933, 8274, 8274, 7959, 8273, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 7933, 8274, 8274, 7959, 8273, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=14848 expected_recv=15905 expected_overflow=0 exact_recv_per_rank=[8276, 7933, 8274, 8274, 7959, 8273, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 7933, 8274, 8274, 7959, 8273, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=14848 expected_recv=16216 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=14848 expected_recv=16209 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 7933, 8274, 8274, 7959, 8273, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7958, 8275, 8275, 7933, 8274, 8274, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=14848 expected_recv=15905 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=14848 expected_recv=15877 expected_overflow=0 exact_recv_per_rank=[7958, 8275, 8275, 7933, 8274, 8274, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=14848 expected_recv=15877 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=14848 expected_recv=16222 expected_overflow=0 exact_recv_per_rank=[7958, 8275, 8275, 7933, 8274, 8274, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[7958, 8275, 8275, 7933, 8274, 8274, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7958, 8275, 8275, 7933, 8274, 8274, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7958, 8275, 8275, 7933, 8274, 8274, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 8274, 8275, 7933, 8275, 7961, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=14848 expected_recv=16222 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=14848 expected_recv=16234 expected_overflow=0 exact_recv_per_rank=[7958, 8275, 8275, 7933, 8274, 8274, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[7958, 8275, 8275, 7933, 8274, 8274, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=14848 expected_recv=16234 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 8274, 8275, 7933, 8275, 7961, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 8274, 8275, 7933, 8275, 7961, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 8274, 8275, 7933, 8275, 7961, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=14848 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 8274, 8275, 7933, 8275, 7961, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8188, 8275, 7926, 8274, 8274, 8051, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=14848 expected_recv=16208 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 8274, 8275, 7933, 8275, 7961, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 8274, 8275, 7933, 8275, 7961, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=14848 expected_recv=15907 expected_overflow=0 exact_recv_per_rank=[8273, 8273, 8274, 8275, 7933, 8275, 7961, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=14848 expected_recv=16545 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=14848 expected_recv=16200 expected_overflow=0 exact_recv_per_rank=[8274, 8188, 8275, 7926, 8274, 8274, 8051, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=14848 expected_recv=16208 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=14848 expected_recv=15907 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=14848 expected_recv=16200 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8188, 8275, 7926, 8274, 8274, 8051, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=14848 expected_recv=16236 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7964, 8274, 7927, 8274, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8188, 8275, 7926, 8274, 8274, 8051, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=14848 expected_recv=16145 expected_overflow=0 exact_recv_per_rank=[8274, 8188, 8275, 7926, 8274, 8274, 8051, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8188, 8275, 7926, 8274, 8274, 8051, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=14848 expected_recv=16236 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=14848 expected_recv=16209 expected_overflow=0 exact_recv_per_rank=[8274, 8188, 8275, 7926, 8274, 8274, 8051, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=14848 expected_recv=16325 expected_overflow=0 exact_recv_per_rank=[8274, 8188, 8275, 7926, 8274, 8274, 8051, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7964, 8274, 7927, 8274, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=14848 expected_recv=16145 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=14848 expected_recv=16209 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=14848 expected_recv=16325 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7964, 8274, 7927, 8274, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 7931, 8274, 7960, 8273, 8275, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7964, 8274, 7927, 8274, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=14848 expected_recv=15871 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7964, 8274, 7927, 8274, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7964, 8274, 7927, 8274, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7964, 8274, 7927, 8274, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=14848 expected_recv=16221 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 7964, 8274, 7927, 8274, 8273, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=14848 expected_recv=16233 expected_overflow=0 exact_recv_per_rank=[8276, 7931, 8274, 7960, 8273, 8275, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=14848 expected_recv=15871 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=14848 expected_recv=16221 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=14848 expected_recv=16233 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 7931, 8274, 7960, 8273, 8275, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8273, 7962, 8273, 8275, 7931, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=14848 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8276, 7931, 8274, 7960, 8273, 8275, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=14848 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8276, 7931, 8274, 7960, 8273, 8275, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=14848 expected_recv=16218 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=14848 expected_recv=16206 expected_overflow=0 exact_recv_per_rank=[8276, 7931, 8274, 7960, 8273, 8275, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 7931, 8274, 7960, 8273, 8275, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 7931, 8274, 7960, 8273, 8275, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=14848 expected_recv=15908 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8273, 7962, 8273, 8275, 7931, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=14848 expected_recv=16217 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=14848 expected_recv=16206 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=14848 expected_recv=15908 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8273, 7962, 8273, 8275, 7931, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8276, 8274, 8275, 7964, 7928, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8273, 7962, 8273, 8275, 7931, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=14848 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8273, 7962, 8273, 8275, 7931, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8273, 7962, 8273, 8275, 7931, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8273, 7962, 8273, 8275, 7931, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8276, 8274, 8275, 7964, 7928, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=14848 expected_recv=15875 expected_overflow=0 exact_recv_per_rank=[8273, 8274, 8273, 7962, 8273, 8275, 7931, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=14848 expected_recv=16545 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=14848 expected_recv=15875 actual_valid_recv=16000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=14848 expected_recv=15912 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8276, 8274, 8275, 7964, 7928, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16000 slack_slots=13696 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=14848 expected_recv=15912 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8274, 8275, 8272, 7928, 8273, 7964] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8276, 8274, 8275, 7964, 7928, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8276, 8274, 8275, 7964, 7928, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8276, 8274, 8275, 7964, 7928, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=14848 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8276, 8274, 8275, 7964, 7928, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8274, 8275, 8272, 7928, 8273, 7964] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=14848 expected_recv=16204 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8276, 8274, 8275, 7964, 7928, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=14848 expected_recv=16545 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=14848 expected_recv=16204 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=14848 expected_recv=15871 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8274, 8275, 8272, 7928, 8273, 7964] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=14848 expected_recv=15871 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=14848 expected_recv=15911 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8274, 8275, 8272, 7928, 8273, 7964] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[28126, 27, 18827, 13036, 5451, 64, 3, 2] avg=8192.0000 max=28126 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.433350 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=14848 expected_recv=15911 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8274, 8275, 8272, 7928, 8273, 7964] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8274, 8275, 8272, 7928, 8273, 7964] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8274, 8275, 8272, 7928, 8273, 7964] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8274, 8275, 8272, 7928, 8273, 7964] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[14064, 27, 14064, 13036, 5451, 64, 4766, 14064] avg=8192.0000 max=14064 max_rank=0 min=27 min_rank=1 imbalance_ratio=1.716797 max_load_reduction=14062 max_load_reduction_pct=0.499964 moved_to_replicas=43176 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[28126, 27, 18827, 13036, 5451, 64, 3, 2] avg=8192.0000 max=28126 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.433350 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[28126, 27, 18827, 13036, 5451, 64, 3, 2] avg=8192.0000 max=28126 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.433350 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[28126, 27, 18827, 13036, 5451, 64, 3, 2] avg=8192.0000 max=28126 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.433350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[14064, 27, 14064, 13036, 5451, 64, 4766, 14064] avg=8192.0000 max=14064 max_rank=0 min=27 min_rank=1 imbalance_ratio=1.716797 max_load_reduction=14062 max_load_reduction_pct=0.499964 moved_to_replicas=43176 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[28126, 27, 18827, 13036, 5451, 64, 3, 2] avg=8192.0000 max=28126 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.433350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[14064, 27, 14064, 13036, 5451, 64, 4766, 14064] avg=8192.0000 max=14064 max_rank=0 min=27 min_rank=1 imbalance_ratio=1.716797 max_load_reduction=14062 max_load_reduction_pct=0.499964 moved_to_replicas=43176 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[5, 3, 32761, 7, 1, 32757, 2, 0] avg=8192.0000 max=32761 max_rank=2 min=0 min_rank=7 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[14064, 27, 14064, 13036, 5451, 64, 4766, 14064] avg=8192.0000 max=14064 max_rank=0 min=27 min_rank=1 imbalance_ratio=1.716797 max_load_reduction=14062 max_load_reduction_pct=0.499964 moved_to_replicas=43176 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[28126, 27, 18827, 13036, 5451, 64, 3, 2] avg=8192.0000 max=28126 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.433350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[28126, 27, 18827, 13036, 5451, 64, 3, 2] avg=8192.0000 max=28126 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.433350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[14064, 27, 14064, 13036, 5451, 64, 4766, 14064] avg=8192.0000 max=14064 max_rank=0 min=27 min_rank=1 imbalance_ratio=1.716797 max_load_reduction=14062 max_load_reduction_pct=0.499964 moved_to_replicas=43176 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[28126, 27, 18827, 13036, 5451, 64, 3, 2] avg=8192.0000 max=28126 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.433350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[8274, 8274, 8275, 7950, 8274, 8274, 7940, 8275] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[5, 3, 32761, 7, 1, 32757, 2, 0] avg=8192.0000 max=32761 max_rank=2 min=0 min_rank=7 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[5, 3, 32761, 7, 1, 32757, 2, 0] avg=8192.0000 max=32761 max_rank=2 min=0 min_rank=7 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[14064, 27, 14064, 13036, 5451, 64, 4766, 14064] avg=8192.0000 max=14064 max_rank=0 min=27 min_rank=1 imbalance_ratio=1.716797 max_load_reduction=14062 max_load_reduction_pct=0.499964 moved_to_replicas=43176 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[14064, 27, 14064, 13036, 5451, 64, 4766, 14064] avg=8192.0000 max=14064 max_rank=0 min=27 min_rank=1 imbalance_ratio=1.716797 max_load_reduction=14062 max_load_reduction_pct=0.499964 moved_to_replicas=43176 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[14064, 27, 14064, 13036, 5451, 64, 4766, 14064] avg=8192.0000 max=14064 max_rank=0 min=27 min_rank=1 imbalance_ratio=1.716797 max_load_reduction=14062 max_load_reduction_pct=0.499964 moved_to_replicas=43176 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[8274, 8274, 8275, 7950, 8274, 8274, 7940, 8275] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[5, 3, 32761, 7, 1, 32757, 2, 0] avg=8192.0000 max=32761 max_rank=2 min=0 min_rank=7 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[8274, 8274, 8275, 7950, 8274, 8274, 7940, 8275] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[5, 3, 32761, 7, 1, 32757, 2, 0] avg=8192.0000 max=32761 max_rank=2 min=0 min_rank=7 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[11, 0, 32756, 1, 0, 32755, 10, 3] avg=8192.0000 max=32756 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[8274, 8274, 8275, 7950, 8274, 8274, 7940, 8275] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[5, 3, 32761, 7, 1, 32757, 2, 0] avg=8192.0000 max=32761 max_rank=2 min=0 min_rank=7 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[5, 3, 32761, 7, 1, 32757, 2, 0] avg=8192.0000 max=32761 max_rank=2 min=0 min_rank=7 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[8274, 8274, 8275, 7950, 8274, 8274, 7940, 8275] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[7956, 8275, 8274, 7936, 8272, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7936 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[5, 3, 32761, 7, 1, 32757, 2, 0] avg=8192.0000 max=32761 max_rank=2 min=0 min_rank=7 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[11, 0, 32756, 1, 0, 32755, 10, 3] avg=8192.0000 max=32756 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[11, 0, 32756, 1, 0, 32755, 10, 3] avg=8192.0000 max=32756 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[8274, 8274, 8275, 7950, 8274, 8274, 7940, 8275] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[8274, 8274, 8275, 7950, 8274, 8274, 7940, 8275] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[7956, 8275, 8274, 7936, 8272, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7936 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[11, 0, 32756, 1, 0, 32755, 10, 3] avg=8192.0000 max=32756 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[8274, 8274, 8275, 7950, 8274, 8274, 7940, 8275] avg=8192.0000 max=8275 max_rank=2 min=7940 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97952 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[7956, 8275, 8274, 7936, 8272, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7936 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[11, 0, 32756, 1, 0, 32755, 10, 3] avg=8192.0000 max=32756 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[9, 32754, 5, 32742, 3, 1, 10, 12] avg=8192.0000 max=32754 max_rank=1 min=1 min_rank=5 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[11, 0, 32756, 1, 0, 32755, 10, 3] avg=8192.0000 max=32756 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[7956, 8275, 8274, 7936, 8272, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7936 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[7956, 8275, 8274, 7936, 8272, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7936 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[11, 0, 32756, 1, 0, 32755, 10, 3] avg=8192.0000 max=32756 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[8273, 8273, 7942, 8274, 8275, 8273, 8275, 7951] avg=8192.0000 max=8275 max_rank=4 min=7942 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[9, 32754, 5, 32742, 3, 1, 10, 12] avg=8192.0000 max=32754 max_rank=1 min=1 min_rank=5 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[11, 0, 32756, 1, 0, 32755, 10, 3] avg=8192.0000 max=32756 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[9, 32754, 5, 32742, 3, 1, 10, 12] avg=8192.0000 max=32754 max_rank=1 min=1 min_rank=5 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[7956, 8275, 8274, 7936, 8272, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7936 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[7956, 8275, 8274, 7936, 8272, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7936 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[8273, 8273, 7942, 8274, 8275, 8273, 8275, 7951] avg=8192.0000 max=8275 max_rank=4 min=7942 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[9, 32754, 5, 32742, 3, 1, 10, 12] avg=8192.0000 max=32754 max_rank=1 min=1 min_rank=5 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[7, 32754, 0, 32754, 8, 1, 5, 7] avg=8192.0000 max=32754 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[9, 32754, 5, 32742, 3, 1, 10, 12] avg=8192.0000 max=32754 max_rank=1 min=1 min_rank=5 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[7956, 8275, 8274, 7936, 8272, 8275, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7936 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24481 max_load_reduction_pct=0.747375 moved_to_replicas=97947 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[8273, 8273, 7942, 8274, 8275, 8273, 8275, 7951] avg=8192.0000 max=8275 max_rank=4 min=7942 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[8273, 8273, 7942, 8274, 8275, 8273, 8275, 7951] avg=8192.0000 max=8275 max_rank=4 min=7942 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[8275, 8276, 8273, 8273, 7953, 8273, 7938, 8275] avg=8192.0000 max=8276 max_rank=1 min=7938 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24478 max_load_reduction_pct=0.747329 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[9, 32754, 5, 32742, 3, 1, 10, 12] avg=8192.0000 max=32754 max_rank=1 min=1 min_rank=5 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[8273, 8273, 7942, 8274, 8275, 8273, 8275, 7951] avg=8192.0000 max=8275 max_rank=4 min=7942 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[9, 32754, 5, 32742, 3, 1, 10, 12] avg=8192.0000 max=32754 max_rank=1 min=1 min_rank=5 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[7, 32754, 0, 32754, 8, 1, 5, 7] avg=8192.0000 max=32754 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[7, 32754, 0, 32754, 8, 1, 5, 7] avg=8192.0000 max=32754 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[9, 32754, 5, 32742, 3, 1, 10, 12] avg=8192.0000 max=32754 max_rank=1 min=1 min_rank=5 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[8273, 8273, 7942, 8274, 8275, 8273, 8275, 7951] avg=8192.0000 max=8275 max_rank=4 min=7942 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[8273, 8273, 7942, 8274, 8275, 8273, 8275, 7951] avg=8192.0000 max=8275 max_rank=4 min=7942 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[8275, 8276, 8273, 8273, 7953, 8273, 7938, 8275] avg=8192.0000 max=8276 max_rank=1 min=7938 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24478 max_load_reduction_pct=0.747329 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[7, 32754, 0, 32754, 8, 1, 5, 7] avg=8192.0000 max=32754 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[32732, 12, 5, 3, 19, 32755, 10, 0] avg=8192.0000 max=32755 max_rank=5 min=0 min_rank=7 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[7, 32754, 0, 32754, 8, 1, 5, 7] avg=8192.0000 max=32754 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[8275, 8276, 8273, 8273, 7953, 8273, 7938, 8275] avg=8192.0000 max=8276 max_rank=1 min=7938 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24478 max_load_reduction_pct=0.747329 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[8273, 8273, 7942, 8274, 8275, 8273, 8275, 7951] avg=8192.0000 max=8275 max_rank=4 min=7942 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97930 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[8275, 8276, 8273, 8273, 7953, 8273, 7938, 8275] avg=8192.0000 max=8276 max_rank=1 min=7938 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24478 max_load_reduction_pct=0.747329 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[8274, 8274, 7941, 8275, 7952, 8275, 8273, 8272] avg=8192.0000 max=8275 max_rank=3 min=7941 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97918 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[7, 32754, 0, 32754, 8, 1, 5, 7] avg=8192.0000 max=32754 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[8275, 8276, 8273, 8273, 7953, 8273, 7938, 8275] avg=8192.0000 max=8276 max_rank=1 min=7938 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24478 max_load_reduction_pct=0.747329 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[7, 32754, 0, 32754, 8, 1, 5, 7] avg=8192.0000 max=32754 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[32732, 12, 5, 3, 19, 32755, 10, 0] avg=8192.0000 max=32755 max_rank=5 min=0 min_rank=7 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[32732, 12, 5, 3, 19, 32755, 10, 0] avg=8192.0000 max=32755 max_rank=5 min=0 min_rank=7 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[7, 32754, 0, 32754, 8, 1, 5, 7] avg=8192.0000 max=32754 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[8275, 8276, 8273, 8273, 7953, 8273, 7938, 8275] avg=8192.0000 max=8276 max_rank=1 min=7938 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24478 max_load_reduction_pct=0.747329 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[8275, 8276, 8273, 8273, 7953, 8273, 7938, 8275] avg=8192.0000 max=8276 max_rank=1 min=7938 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24478 max_load_reduction_pct=0.747329 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[8274, 8274, 7941, 8275, 7952, 8275, 8273, 8272] avg=8192.0000 max=8275 max_rank=3 min=7941 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97918 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[32732, 12, 5, 3, 19, 32755, 10, 0] avg=8192.0000 max=32755 max_rank=5 min=0 min_rank=7 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[7723, 25031, 10, 2, 32753, 0, 4, 13] avg=8192.0000 max=32753 max_rank=4 min=0 min_rank=5 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[8274, 8274, 7941, 8275, 7952, 8275, 8273, 8272] avg=8192.0000 max=8275 max_rank=3 min=7941 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97918 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[32732, 12, 5, 3, 19, 32755, 10, 0] avg=8192.0000 max=32755 max_rank=5 min=0 min_rank=7 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[8275, 8276, 8273, 8273, 7953, 8273, 7938, 8275] avg=8192.0000 max=8276 max_rank=1 min=7938 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24478 max_load_reduction_pct=0.747329 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[8274, 8274, 7941, 8275, 7952, 8275, 8273, 8272] avg=8192.0000 max=8275 max_rank=3 min=7941 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97918 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[7955, 8275, 8273, 8273, 8276, 8275, 7935, 8274] avg=8192.0000 max=8276 max_rank=4 min=7935 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24477 max_load_reduction_pct=0.747321 moved_to_replicas=87298 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[8274, 8274, 7941, 8275, 7952, 8275, 8273, 8272] avg=8192.0000 max=8275 max_rank=3 min=7941 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97918 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[32732, 12, 5, 3, 19, 32755, 10, 0] avg=8192.0000 max=32755 max_rank=5 min=0 min_rank=7 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[7723, 25031, 10, 2, 32753, 0, 4, 13] avg=8192.0000 max=32753 max_rank=4 min=0 min_rank=5 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[32732, 12, 5, 3, 19, 32755, 10, 0] avg=8192.0000 max=32755 max_rank=5 min=0 min_rank=7 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[7723, 25031, 10, 2, 32753, 0, 4, 13] avg=8192.0000 max=32753 max_rank=4 min=0 min_rank=5 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[32732, 12, 5, 3, 19, 32755, 10, 0] avg=8192.0000 max=32755 max_rank=5 min=0 min_rank=7 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[8274, 8274, 7941, 8275, 7952, 8275, 8273, 8272] avg=8192.0000 max=8275 max_rank=3 min=7941 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97918 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[7955, 8275, 8273, 8273, 8276, 8275, 7935, 8274] avg=8192.0000 max=8276 max_rank=4 min=7935 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24477 max_load_reduction_pct=0.747321 moved_to_replicas=87298 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[8274, 8274, 7941, 8275, 7952, 8275, 8273, 8272] avg=8192.0000 max=8275 max_rank=3 min=7941 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97918 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[7723, 25031, 10, 2, 32753, 0, 4, 13] avg=8192.0000 max=32753 max_rank=4 min=0 min_rank=5 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[32754, 1, 13, 0, 14, 2, 32752, 0] avg=8192.0000 max=32754 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[7955, 8275, 8273, 8273, 8276, 8275, 7935, 8274] avg=8192.0000 max=8276 max_rank=4 min=7935 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24477 max_load_reduction_pct=0.747321 moved_to_replicas=87298 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[7723, 25031, 10, 2, 32753, 0, 4, 13] avg=8192.0000 max=32753 max_rank=4 min=0 min_rank=5 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[8274, 8274, 7941, 8275, 7952, 8275, 8273, 8272] avg=8192.0000 max=8275 max_rank=3 min=7941 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24480 max_load_reduction_pct=0.747367 moved_to_replicas=97918 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[7955, 8275, 8273, 8273, 8276, 8275, 7935, 8274] avg=8192.0000 max=8276 max_rank=4 min=7935 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24477 max_load_reduction_pct=0.747321 moved_to_replicas=87298 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[8276, 7933, 8274, 8274, 7959, 8273, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7933 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24478 max_load_reduction_pct=0.747329 moved_to_replicas=97944 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[7955, 8275, 8273, 8273, 8276, 8275, 7935, 8274] avg=8192.0000 max=8276 max_rank=4 min=7935 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24477 max_load_reduction_pct=0.747321 moved_to_replicas=87298 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[7723, 25031, 10, 2, 32753, 0, 4, 13] avg=8192.0000 max=32753 max_rank=4 min=0 min_rank=5 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[32754, 1, 13, 0, 14, 2, 32752, 0] avg=8192.0000 max=32754 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[7723, 25031, 10, 2, 32753, 0, 4, 13] avg=8192.0000 max=32753 max_rank=4 min=0 min_rank=5 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[32754, 1, 13, 0, 14, 2, 32752, 0] avg=8192.0000 max=32754 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[7723, 25031, 10, 2, 32753, 0, 4, 13] avg=8192.0000 max=32753 max_rank=4 min=0 min_rank=5 imbalance_ratio=3.998169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[7955, 8275, 8273, 8273, 8276, 8275, 7935, 8274] avg=8192.0000 max=8276 max_rank=4 min=7935 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24477 max_load_reduction_pct=0.747321 moved_to_replicas=87298 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[8276, 7933, 8274, 8274, 7959, 8273, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7933 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24478 max_load_reduction_pct=0.747329 moved_to_replicas=97944 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[7955, 8275, 8273, 8273, 8276, 8275, 7935, 8274] avg=8192.0000 max=8276 max_rank=4 min=7935 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24477 max_load_reduction_pct=0.747321 moved_to_replicas=87298 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[32754, 1, 13, 0, 14, 2, 32752, 0] avg=8192.0000 max=32754 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[32754, 1, 13, 0, 14, 2, 32752, 0] avg=8192.0000 max=32754 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[13, 32752, 0, 1, 6, 10, 0, 32754] avg=8192.0000 max=32754 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[8276, 7933, 8274, 8274, 7959, 8273, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7933 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24478 max_load_reduction_pct=0.747329 moved_to_replicas=97944 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[7955, 8275, 8273, 8273, 8276, 8275, 7935, 8274] avg=8192.0000 max=8276 max_rank=4 min=7935 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24477 max_load_reduction_pct=0.747321 moved_to_replicas=87298 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[8276, 7933, 8274, 8274, 7959, 8273, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7933 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24478 max_load_reduction_pct=0.747329 moved_to_replicas=97944 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[8276, 7933, 8274, 8274, 7959, 8273, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7933 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24478 max_load_reduction_pct=0.747329 moved_to_replicas=97944 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[32754, 1, 13, 0, 14, 2, 32752, 0] avg=8192.0000 max=32754 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[7958, 8275, 8275, 7933, 8274, 8274, 8275, 8272] avg=8192.0000 max=8275 max_rank=1 min=7933 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97944 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[32754, 1, 13, 0, 14, 2, 32752, 0] avg=8192.0000 max=32754 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[13, 32752, 0, 1, 6, 10, 0, 32754] avg=8192.0000 max=32754 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[13, 32752, 0, 1, 6, 10, 0, 32754] avg=8192.0000 max=32754 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[32754, 1, 13, 0, 14, 2, 32752, 0] avg=8192.0000 max=32754 max_rank=0 min=0 min_rank=3 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[8276, 7933, 8274, 8274, 7959, 8273, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7933 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24478 max_load_reduction_pct=0.747329 moved_to_replicas=97944 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[8276, 7933, 8274, 8274, 7959, 8273, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7933 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24478 max_load_reduction_pct=0.747329 moved_to_replicas=97944 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[13, 32752, 0, 1, 6, 10, 0, 32754] avg=8192.0000 max=32754 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[7958, 8275, 8275, 7933, 8274, 8274, 8275, 8272] avg=8192.0000 max=8275 max_rank=1 min=7933 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97944 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[7958, 8275, 8275, 7933, 8274, 8274, 8275, 8272] avg=8192.0000 max=8275 max_rank=1 min=7933 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97944 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[13, 32752, 0, 1, 6, 10, 0, 32754] avg=8192.0000 max=32754 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[0, 0, 14, 1, 0, 32754, 14, 32753] avg=8192.0000 max=32754 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[8276, 7933, 8274, 8274, 7959, 8273, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7933 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24478 max_load_reduction_pct=0.747329 moved_to_replicas=97944 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[7958, 8275, 8275, 7933, 8274, 8274, 8275, 8272] avg=8192.0000 max=8275 max_rank=1 min=7933 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97944 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[7958, 8275, 8275, 7933, 8274, 8274, 8275, 8272] avg=8192.0000 max=8275 max_rank=1 min=7933 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97944 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[8273, 8273, 8274, 8275, 7933, 8275, 7961, 8272] avg=8192.0000 max=8275 max_rank=3 min=7933 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[13, 32752, 0, 1, 6, 10, 0, 32754] avg=8192.0000 max=32754 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[13, 32752, 0, 1, 6, 10, 0, 32754] avg=8192.0000 max=32754 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[0, 0, 14, 1, 0, 32754, 14, 32753] avg=8192.0000 max=32754 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[0, 0, 14, 1, 0, 32754, 14, 32753] avg=8192.0000 max=32754 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[13, 32752, 0, 1, 6, 10, 0, 32754] avg=8192.0000 max=32754 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[0, 0, 14, 1, 0, 32754, 14, 32753] avg=8192.0000 max=32754 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[7958, 8275, 8275, 7933, 8274, 8274, 8275, 8272] avg=8192.0000 max=8275 max_rank=1 min=7933 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97944 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[7958, 8275, 8275, 7933, 8274, 8274, 8275, 8272] avg=8192.0000 max=8275 max_rank=1 min=7933 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97944 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[8273, 8273, 8274, 8275, 7933, 8275, 7961, 8272] avg=8192.0000 max=8275 max_rank=3 min=7933 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[8273, 8273, 8274, 8275, 7933, 8275, 7961, 8272] avg=8192.0000 max=8275 max_rank=3 min=7933 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[0, 0, 14, 1, 0, 32754, 14, 32753] avg=8192.0000 max=32754 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[15, 8188, 0, 14, 32731, 4, 886, 23698] avg=8192.0000 max=32731 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[7958, 8275, 8275, 7933, 8274, 8274, 8275, 8272] avg=8192.0000 max=8275 max_rank=1 min=7933 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97944 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[8273, 8273, 8274, 8275, 7933, 8275, 7961, 8272] avg=8192.0000 max=8275 max_rank=3 min=7933 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[8273, 8273, 8274, 8275, 7933, 8275, 7961, 8272] avg=8192.0000 max=8275 max_rank=3 min=7933 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[8274, 8188, 8275, 7926, 8274, 8274, 8051, 8274] avg=8192.0000 max=8275 max_rank=2 min=7926 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24456 max_load_reduction_pct=0.747182 moved_to_replicas=82588 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[0, 0, 14, 1, 0, 32754, 14, 32753] avg=8192.0000 max=32754 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[15, 8188, 0, 14, 32731, 4, 886, 23698] avg=8192.0000 max=32731 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[0, 0, 14, 1, 0, 32754, 14, 32753] avg=8192.0000 max=32754 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[15, 8188, 0, 14, 32731, 4, 886, 23698] avg=8192.0000 max=32731 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[0, 0, 14, 1, 0, 32754, 14, 32753] avg=8192.0000 max=32754 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[15, 8188, 0, 14, 32731, 4, 886, 23698] avg=8192.0000 max=32731 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[8273, 8273, 8274, 8275, 7933, 8275, 7961, 8272] avg=8192.0000 max=8275 max_rank=3 min=7933 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[8274, 8188, 8275, 7926, 8274, 8274, 8051, 8274] avg=8192.0000 max=8275 max_rank=2 min=7926 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24456 max_load_reduction_pct=0.747182 moved_to_replicas=82588 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[8273, 8273, 8274, 8275, 7933, 8275, 7961, 8272] avg=8192.0000 max=8275 max_rank=3 min=7933 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[8274, 8188, 8275, 7926, 8274, 8274, 8051, 8274] avg=8192.0000 max=8275 max_rank=2 min=7926 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24456 max_load_reduction_pct=0.747182 moved_to_replicas=82588 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[14, 0, 16, 0, 1, 32748, 11, 32746] avg=8192.0000 max=32748 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[15, 8188, 0, 14, 32731, 4, 886, 23698] avg=8192.0000 max=32731 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[8273, 8273, 8274, 8275, 7933, 8275, 7961, 8272] avg=8192.0000 max=8275 max_rank=3 min=7933 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24479 max_load_reduction_pct=0.747359 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[8274, 8188, 8275, 7926, 8274, 8274, 8051, 8274] avg=8192.0000 max=8275 max_rank=2 min=7926 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24456 max_load_reduction_pct=0.747182 moved_to_replicas=82588 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[8275, 8274, 7964, 8274, 7927, 8274, 8273, 8275] avg=8192.0000 max=8275 max_rank=0 min=7927 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24473 max_load_reduction_pct=0.747313 moved_to_replicas=97928 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[8274, 8188, 8275, 7926, 8274, 8274, 8051, 8274] avg=8192.0000 max=8275 max_rank=2 min=7926 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24456 max_load_reduction_pct=0.747182 moved_to_replicas=82588 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[15, 8188, 0, 14, 32731, 4, 886, 23698] avg=8192.0000 max=32731 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[14, 0, 16, 0, 1, 32748, 11, 32746] avg=8192.0000 max=32748 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[15, 8188, 0, 14, 32731, 4, 886, 23698] avg=8192.0000 max=32731 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[14, 0, 16, 0, 1, 32748, 11, 32746] avg=8192.0000 max=32748 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[15, 8188, 0, 14, 32731, 4, 886, 23698] avg=8192.0000 max=32731 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.995483 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[14, 0, 16, 0, 1, 32748, 11, 32746] avg=8192.0000 max=32748 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[8274, 8188, 8275, 7926, 8274, 8274, 8051, 8274] avg=8192.0000 max=8275 max_rank=2 min=7926 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24456 max_load_reduction_pct=0.747182 moved_to_replicas=82588 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[8275, 8274, 7964, 8274, 7927, 8274, 8273, 8275] avg=8192.0000 max=8275 max_rank=0 min=7927 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24473 max_load_reduction_pct=0.747313 moved_to_replicas=97928 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[8274, 8188, 8275, 7926, 8274, 8274, 8051, 8274] avg=8192.0000 max=8275 max_rank=2 min=7926 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24456 max_load_reduction_pct=0.747182 moved_to_replicas=82588 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[8275, 8274, 7964, 8274, 7927, 8274, 8273, 8275] avg=8192.0000 max=8275 max_rank=0 min=7927 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24473 max_load_reduction_pct=0.747313 moved_to_replicas=97928 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[0, 2, 32752, 16, 0, 32750, 11, 5] avg=8192.0000 max=32752 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[14, 0, 16, 0, 1, 32748, 11, 32746] avg=8192.0000 max=32748 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[8275, 8274, 7964, 8274, 7927, 8274, 8273, 8275] avg=8192.0000 max=8275 max_rank=0 min=7927 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24473 max_load_reduction_pct=0.747313 moved_to_replicas=97928 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[8274, 8188, 8275, 7926, 8274, 8274, 8051, 8274] avg=8192.0000 max=8275 max_rank=2 min=7926 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24456 max_load_reduction_pct=0.747182 moved_to_replicas=82588 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[8276, 7931, 8274, 7960, 8273, 8275, 8274, 8273] avg=8192.0000 max=8276 max_rank=0 min=7931 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24476 max_load_reduction_pct=0.747313 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[8275, 8274, 7964, 8274, 7927, 8274, 8273, 8275] avg=8192.0000 max=8275 max_rank=0 min=7927 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24473 max_load_reduction_pct=0.747313 moved_to_replicas=97928 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[14, 0, 16, 0, 1, 32748, 11, 32746] avg=8192.0000 max=32748 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[0, 2, 32752, 16, 0, 32750, 11, 5] avg=8192.0000 max=32752 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[14, 0, 16, 0, 1, 32748, 11, 32746] avg=8192.0000 max=32748 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[0, 2, 32752, 16, 0, 32750, 11, 5] avg=8192.0000 max=32752 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[0, 2, 32752, 16, 0, 32750, 11, 5] avg=8192.0000 max=32752 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[14, 0, 16, 0, 1, 32748, 11, 32746] avg=8192.0000 max=32748 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.997559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[8275, 8274, 7964, 8274, 7927, 8274, 8273, 8275] avg=8192.0000 max=8275 max_rank=0 min=7927 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24473 max_load_reduction_pct=0.747313 moved_to_replicas=97928 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[8276, 7931, 8274, 7960, 8273, 8275, 8274, 8273] avg=8192.0000 max=8276 max_rank=0 min=7931 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24476 max_load_reduction_pct=0.747313 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[8275, 8274, 7964, 8274, 7927, 8274, 8273, 8275] avg=8192.0000 max=8275 max_rank=0 min=7927 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24473 max_load_reduction_pct=0.747313 moved_to_replicas=97928 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[32752, 32744, 13, 24, 0, 0, 0, 3] avg=8192.0000 max=32752 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[0, 2, 32752, 16, 0, 32750, 11, 5] avg=8192.0000 max=32752 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[8276, 7931, 8274, 7960, 8273, 8275, 8274, 8273] avg=8192.0000 max=8276 max_rank=0 min=7931 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24476 max_load_reduction_pct=0.747313 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[8276, 7931, 8274, 7960, 8273, 8275, 8274, 8273] avg=8192.0000 max=8276 max_rank=0 min=7931 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24476 max_load_reduction_pct=0.747313 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[8275, 8274, 7964, 8274, 7927, 8274, 8273, 8275] avg=8192.0000 max=8275 max_rank=0 min=7927 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24473 max_load_reduction_pct=0.747313 moved_to_replicas=97928 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[8273, 8274, 8273, 7962, 8273, 8275, 7931, 8275] avg=8192.0000 max=8275 max_rank=5 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97934 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[8276, 7931, 8274, 7960, 8273, 8275, 8274, 8273] avg=8192.0000 max=8276 max_rank=0 min=7931 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24476 max_load_reduction_pct=0.747313 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[32752, 32744, 13, 24, 0, 0, 0, 3] avg=8192.0000 max=32752 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[0, 2, 32752, 16, 0, 32750, 11, 5] avg=8192.0000 max=32752 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[0, 2, 32752, 16, 0, 32750, 11, 5] avg=8192.0000 max=32752 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[32752, 32744, 13, 24, 0, 0, 0, 3] avg=8192.0000 max=32752 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[32752, 32744, 13, 24, 0, 0, 0, 3] avg=8192.0000 max=32752 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[0, 2, 32752, 16, 0, 32750, 11, 5] avg=8192.0000 max=32752 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[8273, 8274, 8273, 7962, 8273, 8275, 7931, 8275] avg=8192.0000 max=8275 max_rank=5 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97934 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[8276, 7931, 8274, 7960, 8273, 8275, 8274, 8273] avg=8192.0000 max=8276 max_rank=0 min=7931 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24476 max_load_reduction_pct=0.747313 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[8276, 7931, 8274, 7960, 8273, 8275, 8274, 8273] avg=8192.0000 max=8276 max_rank=0 min=7931 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24476 max_load_reduction_pct=0.747313 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[4, 16, 0, 32748, 0, 17, 0, 32751] avg=8192.0000 max=32751 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[32752, 32744, 13, 24, 0, 0, 0, 3] avg=8192.0000 max=32752 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[8273, 8274, 8273, 7962, 8273, 8275, 7931, 8275] avg=8192.0000 max=8275 max_rank=5 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97934 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[8273, 8274, 8273, 7962, 8273, 8275, 7931, 8275] avg=8192.0000 max=8275 max_rank=5 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97934 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[8276, 7931, 8274, 7960, 8273, 8275, 8274, 8273] avg=8192.0000 max=8276 max_rank=0 min=7931 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24476 max_load_reduction_pct=0.747313 moved_to_replicas=97940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[8275, 8272, 8276, 8274, 8275, 7964, 7928, 8272] avg=8192.0000 max=8276 max_rank=2 min=7928 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24475 max_load_reduction_pct=0.747305 moved_to_replicas=97937 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[8273, 8274, 8273, 7962, 8273, 8275, 7931, 8275] avg=8192.0000 max=8275 max_rank=5 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97934 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[4, 16, 0, 32748, 0, 17, 0, 32751] avg=8192.0000 max=32751 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[32752, 32744, 13, 24, 0, 0, 0, 3] avg=8192.0000 max=32752 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[32752, 32744, 13, 24, 0, 0, 0, 3] avg=8192.0000 max=32752 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[4, 16, 0, 32748, 0, 17, 0, 32751] avg=8192.0000 max=32751 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[4, 16, 0, 32748, 0, 17, 0, 32751] avg=8192.0000 max=32751 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[32752, 32744, 13, 24, 0, 0, 0, 3] avg=8192.0000 max=32752 max_rank=0 min=0 min_rank=4 imbalance_ratio=3.998047 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[8275, 8272, 8276, 8274, 8275, 7964, 7928, 8272] avg=8192.0000 max=8276 max_rank=2 min=7928 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24475 max_load_reduction_pct=0.747305 moved_to_replicas=97937 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[17, 32750, 32555, 0, 0, 1, 19, 194] avg=8192.0000 max=32750 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[8273, 8274, 8273, 7962, 8273, 8275, 7931, 8275] avg=8192.0000 max=8275 max_rank=5 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97934 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[8273, 8274, 8273, 7962, 8273, 8275, 7931, 8275] avg=8192.0000 max=8275 max_rank=5 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97934 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[4, 16, 0, 32748, 0, 17, 0, 32751] avg=8192.0000 max=32751 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[8275, 8272, 8276, 8274, 8275, 7964, 7928, 8272] avg=8192.0000 max=8276 max_rank=2 min=7928 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24475 max_load_reduction_pct=0.747305 moved_to_replicas=97937 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[8275, 8272, 8276, 8274, 8275, 7964, 7928, 8272] avg=8192.0000 max=8276 max_rank=2 min=7928 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24475 max_load_reduction_pct=0.747305 moved_to_replicas=97937 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[8273, 8274, 8273, 7962, 8273, 8275, 7931, 8275] avg=8192.0000 max=8275 max_rank=5 min=7931 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24477 max_load_reduction_pct=0.747344 moved_to_replicas=97934 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[8274, 8276, 8274, 8275, 8272, 7928, 8273, 7964] avg=8192.0000 max=8276 max_rank=1 min=7928 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24474 max_load_reduction_pct=0.747298 moved_to_replicas=97733 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[8275, 8272, 8276, 8274, 8275, 7964, 7928, 8272] avg=8192.0000 max=8276 max_rank=2 min=7928 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24475 max_load_reduction_pct=0.747305 moved_to_replicas=97937 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[17, 32750, 32555, 0, 0, 1, 19, 194] avg=8192.0000 max=32750 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[4, 16, 0, 32748, 0, 17, 0, 32751] avg=8192.0000 max=32751 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[4, 16, 0, 32748, 0, 17, 0, 32751] avg=8192.0000 max=32751 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[17, 32750, 32555, 0, 0, 1, 19, 194] avg=8192.0000 max=32750 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[17, 32750, 32555, 0, 0, 1, 19, 194] avg=8192.0000 max=32750 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[4, 16, 0, 32748, 0, 17, 0, 32751] avg=8192.0000 max=32751 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.997925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[8274, 8276, 8274, 8275, 8272, 7928, 8273, 7964] avg=8192.0000 max=8276 max_rank=1 min=7928 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24474 max_load_reduction_pct=0.747298 moved_to_replicas=97733 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[8275, 8272, 8276, 8274, 8275, 7964, 7928, 8272] avg=8192.0000 max=8276 max_rank=2 min=7928 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24475 max_load_reduction_pct=0.747305 moved_to_replicas=97937 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[8275, 8272, 8276, 8274, 8275, 7964, 7928, 8272] avg=8192.0000 max=8276 max_rank=2 min=7928 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24475 max_load_reduction_pct=0.747305 moved_to_replicas=97937 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[8274, 8276, 8274, 8275, 8272, 7928, 8273, 7964] avg=8192.0000 max=8276 max_rank=1 min=7928 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24474 max_load_reduction_pct=0.747298 moved_to_replicas=97733 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[8274, 8276, 8274, 8275, 8272, 7928, 8273, 7964] avg=8192.0000 max=8276 max_rank=1 min=7928 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24474 max_load_reduction_pct=0.747298 moved_to_replicas=97733 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[17, 32750, 32555, 0, 0, 1, 19, 194] avg=8192.0000 max=32750 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[8275, 8272, 8276, 8274, 8275, 7964, 7928, 8272] avg=8192.0000 max=8276 max_rank=2 min=7928 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24475 max_load_reduction_pct=0.747305 moved_to_replicas=97937 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[8274, 8276, 8274, 8275, 8272, 7928, 8273, 7964] avg=8192.0000 max=8276 max_rank=1 min=7928 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24474 max_load_reduction_pct=0.747298 moved_to_replicas=97733 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[17, 32750, 32555, 0, 0, 1, 19, 194] avg=8192.0000 max=32750 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[17, 32750, 32555, 0, 0, 1, 19, 194] avg=8192.0000 max=32750 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[17, 32750, 32555, 0, 0, 1, 19, 194] avg=8192.0000 max=32750 max_rank=1 min=0 min_rank=3 imbalance_ratio=3.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[8274, 8276, 8274, 8275, 8272, 7928, 8273, 7964] avg=8192.0000 max=8276 max_rank=1 min=7928 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24474 max_load_reduction_pct=0.747298 moved_to_replicas=97733 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[8274, 8276, 8274, 8275, 8272, 7928, 8273, 7964] avg=8192.0000 max=8276 max_rank=1 min=7928 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24474 max_load_reduction_pct=0.747298 moved_to_replicas=97733 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[8274, 8276, 8274, 8275, 8272, 7928, 8273, 7964] avg=8192.0000 max=8276 max_rank=1 min=7928 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24474 max_load_reduction_pct=0.747298 moved_to_replicas=97733 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=32 total_expected_recv=271222 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=200448 peak_demand=14064 required_capacity=1.716797 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=32 total_expected_recv=268543 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=202880 peak_demand=13036 required_capacity=1.591309 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=32 total_expected_recv=249579 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=222336 peak_demand=8276 required_capacity=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=32 total_expected_recv=265500 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=206208 peak_demand=14064 required_capacity=1.716797 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=32 total_expected_recv=260292 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=211456 peak_demand=10249 required_capacity=1.251099 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=32 total_expected_recv=252699 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=219648 peak_demand=8276 required_capacity=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=32 total_expected_recv=270602 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=200832 peak_demand=14064 required_capacity=1.716797 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=32 total_expected_recv=258715 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=211968 peak_demand=10250 required_capacity=1.251221 + [2026-08-30 05:09:17] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 910.4 | throughput per GPU (TFLOP/s/GPU): 366.2 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.045477E+00 | loss scale: 1.0 | grad norm: 6.972 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 14208 | min safe capacity: 1.7344 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=14848 expected_recv=18789 expected_overflow=0 exact_recv_per_rank=[9758, 9759, 4728, 9757, 9758, 6508, 5510, 9758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=14848 expected_recv=9149 expected_overflow=0 exact_recv_per_rank=[9758, 9759, 4728, 9757, 9758, 6508, 5510, 9758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=14848 expected_recv=18788 expected_overflow=0 exact_recv_per_rank=[9758, 9759, 4728, 9757, 9758, 6508, 5510, 9758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=14848 expected_recv=15469 expected_overflow=0 exact_recv_per_rank=[9758, 9759, 4728, 9757, 9758, 6508, 5510, 9758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=14848 expected_recv=18790 expected_overflow=0 exact_recv_per_rank=[9758, 9759, 4728, 9757, 9758, 6508, 5510, 9758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=14848 expected_recv=16757 expected_overflow=0 exact_recv_per_rank=[9758, 9759, 4728, 9757, 9758, 6508, 5510, 9758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=14848 expected_recv=18789 actual_valid_recv=19072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=14848 expected_recv=9149 actual_valid_recv=9216 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=14848 expected_recv=18788 actual_valid_recv=18944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=14848 expected_recv=14542 expected_overflow=0 exact_recv_per_rank=[9758, 9759, 4728, 9757, 9758, 6508, 5510, 9758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=14848 expected_recv=18788 expected_overflow=0 exact_recv_per_rank=[9758, 9759, 4728, 9757, 9758, 6508, 5510, 9758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=14848 expected_recv=15469 actual_valid_recv=15744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=19072 slack_slots=10624 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=9216 slack_slots=20480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=14848 expected_recv=18790 actual_valid_recv=18944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=18944 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=14848 expected_recv=16757 actual_valid_recv=17024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=15744 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=18944 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=14848 expected_recv=14542 actual_valid_recv=14976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=17024 slack_slots=12672 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=14848 expected_recv=18788 actual_valid_recv=18944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=14976 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=18944 slack_slots=10752 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7939, 8274, 8273, 8274, 8274, 7954, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7939, 8274, 8273, 8274, 8274, 7954, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7939, 8274, 8273, 8274, 8274, 7954, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=14848 expected_recv=15901 expected_overflow=0 exact_recv_per_rank=[7939, 8274, 8273, 8274, 8274, 7954, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7939, 8274, 8273, 8274, 8274, 7954, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=14848 expected_recv=15883 expected_overflow=0 exact_recv_per_rank=[7939, 8274, 8273, 8274, 8274, 7954, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=14848 expected_recv=15901 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=14848 expected_recv=15883 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[7939, 8274, 8273, 8274, 8274, 7954, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7939, 8274, 8273, 8274, 8274, 7954, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 8274, 7945, 8275, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 8274, 7945, 8275, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 8274, 7945, 8275, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 8274, 7945, 8275, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=14848 expected_recv=16221 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 8274, 7945, 8275, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 8274, 7945, 8275, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=14848 expected_recv=16221 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=14848 expected_recv=15889 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 8274, 7945, 8275, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8273, 8275, 8275, 8274, 7945, 8275, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=14848 expected_recv=15889 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8274, 8275, 8272, 7947, 8275, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8274, 8275, 8272, 7947, 8275, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8274, 8275, 8272, 7947, 8275, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=14848 expected_recv=15894 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8274, 8275, 8272, 7947, 8275, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=14848 expected_recv=15889 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8274, 8275, 8272, 7947, 8275, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8274, 8275, 8272, 7947, 8275, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8274, 8275, 8272, 7947, 8275, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=14848 expected_recv=15894 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=14848 expected_recv=15889 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8274, 8275, 8272, 7947, 8275, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 8274, 8275, 7946, 8272, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 8274, 8275, 7946, 8272, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 8274, 8275, 7946, 8272, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 8274, 8275, 7946, 8272, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 8274, 8275, 7946, 8272, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=14848 expected_recv=16221 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 8274, 8275, 7946, 8272, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=14848 expected_recv=15890 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 8274, 8275, 7946, 8272, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 8274, 8275, 7946, 8272, 8275, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=14848 expected_recv=16221 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=14848 expected_recv=15890 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 7944, 7948, 8274, 8275, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 7944, 7948, 8274, 8275, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=14848 expected_recv=15894 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 7944, 7948, 8274, 8275, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=14848 expected_recv=15894 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=14848 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 7944, 7948, 8274, 8275, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 7944, 7948, 8274, 8275, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 7944, 7948, 8274, 8275, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 7944, 7948, 8274, 8275, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=14848 expected_recv=16216 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 7944, 7948, 8274, 8275, 8273, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=14848 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8272, 7948, 7944, 8276, 8275, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=14848 expected_recv=16218 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=14848 expected_recv=15894 expected_overflow=0 exact_recv_per_rank=[8272, 7948, 7944, 8276, 8275, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8272, 7948, 7944, 8276, 8275, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=14848 expected_recv=15894 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8272, 7948, 7944, 8276, 8275, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8272, 7948, 7944, 8276, 8275, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8272, 7948, 7944, 8276, 8275, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8272, 7948, 7944, 8276, 8275, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8272, 7948, 7944, 8276, 8275, 8272, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8275, 8273, 7947, 8275, 8275, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=14848 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8275, 8273, 7947, 8275, 8275, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=14848 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8275, 8273, 7947, 8275, 8275, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=14848 expected_recv=16545 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=14848 expected_recv=16545 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8275, 8273, 7947, 8275, 8275, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8275, 8273, 7947, 8275, 8275, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=14848 expected_recv=15893 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8275, 8273, 7947, 8275, 8275, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8275, 8273, 7947, 8275, 8275, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=14848 expected_recv=15893 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7950, 8276, 8275, 8272, 8273, 8273, 7943, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8275, 8272, 8275, 8273, 7947, 8275, 8275, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7950, 8276, 8275, 8272, 8273, 8273, 7943, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=14848 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[7950, 8276, 8275, 8272, 8273, 8273, 7943, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=14848 expected_recv=16216 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7950, 8276, 8275, 8272, 8273, 8273, 7943, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[7950, 8276, 8275, 8272, 8273, 8273, 7943, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[7950, 8276, 8275, 8272, 8273, 8273, 7943, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=14848 expected_recv=16226 expected_overflow=0 exact_recv_per_rank=[7950, 8276, 8275, 8272, 8273, 8273, 7943, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8273, 8272, 7944, 8276, 7949, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=14848 expected_recv=16226 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=14848 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[7950, 8276, 8275, 8272, 8273, 8273, 7943, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8273, 8272, 7944, 8276, 7949, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=14848 expected_recv=16218 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8273, 8272, 7944, 8276, 7949, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8273, 8272, 7944, 8276, 7949, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8273, 8272, 7944, 8276, 7949, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=14848 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8273, 8272, 7944, 8276, 7949, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8273, 8272, 7944, 8276, 7949, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=14848 expected_recv=15888 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7950, 8272, 8275, 8273, 8274, 8273, 8275, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=14848 expected_recv=15894 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8273, 8272, 7944, 8276, 7949, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7950, 8272, 8275, 8273, 8274, 8273, 8275, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=14848 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[7950, 8272, 8275, 8273, 8274, 8273, 8275, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=14848 expected_recv=15894 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[7950, 8272, 8275, 8273, 8274, 8273, 8275, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=14848 expected_recv=16545 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[7950, 8272, 8275, 8273, 8274, 8273, 8275, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7950, 8272, 8275, 8273, 8274, 8273, 8275, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=14848 expected_recv=16223 expected_overflow=0 exact_recv_per_rank=[7950, 8272, 8275, 8273, 8274, 8273, 8275, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=14848 expected_recv=16223 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=14848 expected_recv=15897 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 7949, 8274, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[7950, 8272, 8275, 8273, 8274, 8273, 8275, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=14848 expected_recv=15897 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 7949, 8274, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 7949, 8274, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 7949, 8274, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 7949, 8274, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=14848 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 7949, 8274, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 7949, 8274, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=14848 expected_recv=15888 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8274, 8272, 8274, 7943, 7948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 7949, 8274, 7944, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8274, 8272, 8274, 7943, 7948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8274, 8272, 8274, 7943, 7948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=14848 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8274, 8272, 8274, 7943, 7948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=14848 expected_recv=16220 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8274, 8272, 8274, 7943, 7948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8274, 8272, 8274, 7943, 7948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=14848 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8274, 8272, 8274, 7943, 7948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7947, 8275, 8272, 7943, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=14848 expected_recv=16220 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=14848 expected_recv=15887 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8274, 8272, 8274, 7943, 7948] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=14848 expected_recv=15896 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7947, 8275, 8272, 7943, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=14848 expected_recv=15887 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7947, 8275, 8272, 7943, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7947, 8275, 8272, 7943, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=14848 expected_recv=15896 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7947, 8275, 8272, 7943, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7947, 8275, 8272, 7943, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7947, 8275, 8272, 7943, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8275, 8275, 7945, 8273, 7948, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=14848 expected_recv=15887 expected_overflow=0 exact_recv_per_rank=[8276, 8275, 8273, 7947, 8275, 8272, 7943, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8275, 8275, 7945, 8273, 7948, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8275, 8275, 7945, 8273, 7948, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=14848 expected_recv=15887 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8275, 8275, 7945, 8273, 7948, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8275, 8275, 7945, 8273, 7948, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=14848 expected_recv=15889 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8275, 8275, 7945, 8273, 7948, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8275, 8275, 7945, 8273, 7948, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=14848 expected_recv=15889 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7947, 8275, 8275, 8272, 7944, 8273, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=14848 expected_recv=15896 expected_overflow=0 exact_recv_per_rank=[8275, 8273, 8275, 8275, 7945, 8273, 7948, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[7947, 8275, 8275, 8272, 7944, 8273, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[7947, 8275, 8275, 8272, 7944, 8273, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=14848 expected_recv=15896 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7947, 8275, 8275, 8272, 7944, 8273, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[7947, 8275, 8275, 8272, 7944, 8273, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=14848 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[7947, 8275, 8275, 8272, 7944, 8273, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=14848 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[7947, 8275, 8275, 8272, 7944, 8273, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=14848 expected_recv=15888 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=14848 expected_recv=16220 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7947, 8275, 8275, 8272, 7944, 8273, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[15061, 31, 4728, 19485, 25023, 1205, 3, 0] avg=8192.0000 max=25023 max_rank=4 min=0 min_rank=7 imbalance_ratio=3.054565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[15061, 31, 4728, 19485, 25023, 1205, 3, 0] avg=8192.0000 max=25023 max_rank=4 min=0 min_rank=7 imbalance_ratio=3.054565 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[15061, 31, 4728, 19485, 25023, 1205, 3, 0] avg=8192.0000 max=25023 max_rank=4 min=0 min_rank=7 imbalance_ratio=3.054565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[15061, 31, 4728, 19485, 25023, 1205, 3, 0] avg=8192.0000 max=25023 max_rank=4 min=0 min_rank=7 imbalance_ratio=3.054565 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[9758, 9759, 4728, 9757, 9758, 6508, 5510, 9758] avg=8192.0000 max=9759 max_rank=1 min=4728 min_rank=2 imbalance_ratio=1.191284 max_load_reduction=15264 max_load_reduction_pct=0.609999 moved_to_replicas=63459 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[15061, 31, 4728, 19485, 25023, 1205, 3, 0] avg=8192.0000 max=25023 max_rank=4 min=0 min_rank=7 imbalance_ratio=3.054565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[15061, 31, 4728, 19485, 25023, 1205, 3, 0] avg=8192.0000 max=25023 max_rank=4 min=0 min_rank=7 imbalance_ratio=3.054565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[9758, 9759, 4728, 9757, 9758, 6508, 5510, 9758] avg=8192.0000 max=9759 max_rank=1 min=4728 min_rank=2 imbalance_ratio=1.191284 max_load_reduction=15264 max_load_reduction_pct=0.609999 moved_to_replicas=63459 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[9758, 9759, 4728, 9757, 9758, 6508, 5510, 9758] avg=8192.0000 max=9759 max_rank=1 min=4728 min_rank=2 imbalance_ratio=1.191284 max_load_reduction=15264 max_load_reduction_pct=0.609999 moved_to_replicas=63459 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[15061, 31, 4728, 19485, 25023, 1205, 3, 0] avg=8192.0000 max=25023 max_rank=4 min=0 min_rank=7 imbalance_ratio=3.054565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[9758, 9759, 4728, 9757, 9758, 6508, 5510, 9758] avg=8192.0000 max=9759 max_rank=1 min=4728 min_rank=2 imbalance_ratio=1.191284 max_load_reduction=15264 max_load_reduction_pct=0.609999 moved_to_replicas=63459 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[9758, 9759, 4728, 9757, 9758, 6508, 5510, 9758] avg=8192.0000 max=9759 max_rank=1 min=4728 min_rank=2 imbalance_ratio=1.191284 max_load_reduction=15264 max_load_reduction_pct=0.609999 moved_to_replicas=63459 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[15061, 31, 4728, 19485, 25023, 1205, 3, 0] avg=8192.0000 max=25023 max_rank=4 min=0 min_rank=7 imbalance_ratio=3.054565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[1, 32714, 32759, 0, 0, 52, 2, 8] avg=8192.0000 max=32759 max_rank=2 min=0 min_rank=3 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[9758, 9759, 4728, 9757, 9758, 6508, 5510, 9758] avg=8192.0000 max=9759 max_rank=1 min=4728 min_rank=2 imbalance_ratio=1.191284 max_load_reduction=15264 max_load_reduction_pct=0.609999 moved_to_replicas=63459 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[9758, 9759, 4728, 9757, 9758, 6508, 5510, 9758] avg=8192.0000 max=9759 max_rank=1 min=4728 min_rank=2 imbalance_ratio=1.191284 max_load_reduction=15264 max_load_reduction_pct=0.609999 moved_to_replicas=63459 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[1, 32714, 32759, 0, 0, 52, 2, 8] avg=8192.0000 max=32759 max_rank=2 min=0 min_rank=3 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[1, 32714, 32759, 0, 0, 52, 2, 8] avg=8192.0000 max=32759 max_rank=2 min=0 min_rank=3 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[1, 32714, 32759, 0, 0, 52, 2, 8] avg=8192.0000 max=32759 max_rank=2 min=0 min_rank=3 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[9758, 9759, 4728, 9757, 9758, 6508, 5510, 9758] avg=8192.0000 max=9759 max_rank=1 min=4728 min_rank=2 imbalance_ratio=1.191284 max_load_reduction=15264 max_load_reduction_pct=0.609999 moved_to_replicas=63459 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[7939, 8274, 8273, 8274, 8274, 7954, 8274, 8274] avg=8192.0000 max=8274 max_rank=1 min=7939 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24485 max_load_reduction_pct=0.747428 moved_to_replicas=97883 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[1, 32714, 32759, 0, 0, 52, 2, 8] avg=8192.0000 max=32759 max_rank=2 min=0 min_rank=3 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[7939, 8274, 8273, 8274, 8274, 7954, 8274, 8274] avg=8192.0000 max=8274 max_rank=1 min=7939 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24485 max_load_reduction_pct=0.747428 moved_to_replicas=97883 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[1, 32714, 32759, 0, 0, 52, 2, 8] avg=8192.0000 max=32759 max_rank=2 min=0 min_rank=3 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[7939, 8274, 8273, 8274, 8274, 7954, 8274, 8274] avg=8192.0000 max=8274 max_rank=1 min=7939 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24485 max_load_reduction_pct=0.747428 moved_to_replicas=97883 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[7939, 8274, 8273, 8274, 8274, 7954, 8274, 8274] avg=8192.0000 max=8274 max_rank=1 min=7939 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24485 max_load_reduction_pct=0.747428 moved_to_replicas=97883 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[1, 32714, 32759, 0, 0, 52, 2, 8] avg=8192.0000 max=32759 max_rank=2 min=0 min_rank=3 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[7939, 8274, 8273, 8274, 8274, 7954, 8274, 8274] avg=8192.0000 max=8274 max_rank=1 min=7939 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24485 max_load_reduction_pct=0.747428 moved_to_replicas=97883 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[1, 0, 32766, 2, 1, 32763, 1, 2] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[1, 32714, 32759, 0, 0, 52, 2, 8] avg=8192.0000 max=32759 max_rank=2 min=0 min_rank=3 imbalance_ratio=3.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[7939, 8274, 8273, 8274, 8274, 7954, 8274, 8274] avg=8192.0000 max=8274 max_rank=1 min=7939 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24485 max_load_reduction_pct=0.747428 moved_to_replicas=97883 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[7939, 8274, 8273, 8274, 8274, 7954, 8274, 8274] avg=8192.0000 max=8274 max_rank=1 min=7939 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24485 max_load_reduction_pct=0.747428 moved_to_replicas=97883 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[1, 0, 32766, 2, 1, 32763, 1, 2] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[1, 0, 32766, 2, 1, 32763, 1, 2] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[1, 0, 32766, 2, 1, 32763, 1, 2] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[8273, 8275, 8275, 8274, 7945, 8275, 8273, 7946] avg=8192.0000 max=8275 max_rank=1 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[7939, 8274, 8273, 8274, 8274, 7954, 8274, 8274] avg=8192.0000 max=8274 max_rank=1 min=7939 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24485 max_load_reduction_pct=0.747428 moved_to_replicas=97883 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[1, 0, 32766, 2, 1, 32763, 1, 2] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[1, 0, 32766, 2, 1, 32763, 1, 2] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[8273, 8275, 8275, 8274, 7945, 8275, 8273, 7946] avg=8192.0000 max=8275 max_rank=1 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[8273, 8275, 8275, 8274, 7945, 8275, 8273, 7946] avg=8192.0000 max=8275 max_rank=1 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[8273, 8275, 8275, 8274, 7945, 8275, 8273, 7946] avg=8192.0000 max=8275 max_rank=1 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[1, 0, 32766, 2, 1, 32763, 1, 2] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[8273, 8275, 8275, 8274, 7945, 8275, 8273, 7946] avg=8192.0000 max=8275 max_rank=1 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[0, 3, 2, 32658, 0, 106, 32766, 1] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[8273, 8275, 8275, 8274, 7945, 8275, 8273, 7946] avg=8192.0000 max=8275 max_rank=1 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[1, 0, 32766, 2, 1, 32763, 1, 2] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[8273, 8275, 8275, 8274, 7945, 8275, 8273, 7946] avg=8192.0000 max=8275 max_rank=1 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[0, 3, 2, 32658, 0, 106, 32766, 1] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[0, 3, 2, 32658, 0, 106, 32766, 1] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[0, 3, 2, 32658, 0, 106, 32766, 1] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[8275, 8273, 8274, 8275, 8272, 7947, 8275, 7945] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97807 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[8273, 8275, 8275, 8274, 7945, 8275, 8273, 7946] avg=8192.0000 max=8275 max_rank=1 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[0, 3, 2, 32658, 0, 106, 32766, 1] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[0, 3, 2, 32658, 0, 106, 32766, 1] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[8275, 8273, 8274, 8275, 8272, 7947, 8275, 7945] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97807 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[8275, 8273, 8274, 8275, 8272, 7947, 8275, 7945] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97807 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[8275, 8273, 8274, 8275, 8272, 7947, 8275, 7945] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97807 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[0, 3, 2, 32658, 0, 106, 32766, 1] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[8275, 8273, 8274, 8275, 8272, 7947, 8275, 7945] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97807 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[1, 1, 32765, 32766, 2, 0, 0, 1] avg=8192.0000 max=32766 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[8275, 8273, 8274, 8275, 8272, 7947, 8275, 7945] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97807 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[0, 3, 2, 32658, 0, 106, 32766, 1] avg=8192.0000 max=32766 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[8275, 8273, 8274, 8275, 8272, 7947, 8275, 7945] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97807 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[1, 1, 32765, 32766, 2, 0, 0, 1] avg=8192.0000 max=32766 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[1, 1, 32765, 32766, 2, 0, 0, 1] avg=8192.0000 max=32766 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[1, 1, 32765, 32766, 2, 0, 0, 1] avg=8192.0000 max=32766 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[7945, 8276, 8274, 8275, 7946, 8272, 8275, 8273] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97969 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[8275, 8273, 8274, 8275, 8272, 7947, 8275, 7945] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97807 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[1, 1, 32765, 32766, 2, 0, 0, 1] avg=8192.0000 max=32766 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[1, 1, 32765, 32766, 2, 0, 0, 1] avg=8192.0000 max=32766 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[7945, 8276, 8274, 8275, 7946, 8272, 8275, 8273] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97969 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[7945, 8276, 8274, 8275, 7946, 8272, 8275, 8273] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97969 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[7945, 8276, 8274, 8275, 7946, 8272, 8275, 8273] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97969 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[1, 1, 32765, 32766, 2, 0, 0, 1] avg=8192.0000 max=32766 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[7945, 8276, 8274, 8275, 7946, 8272, 8275, 8273] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97969 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[0, 2, 1, 12, 32754, 32766, 1, 0] avg=8192.0000 max=32766 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[7945, 8276, 8274, 8275, 7946, 8272, 8275, 8273] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97969 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[1, 1, 32765, 32766, 2, 0, 0, 1] avg=8192.0000 max=32766 max_rank=3 min=0 min_rank=5 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[7945, 8276, 8274, 8275, 7946, 8272, 8275, 8273] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97969 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[8276, 8274, 7944, 7948, 8274, 8275, 8273, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[0, 2, 1, 12, 32754, 32766, 1, 0] avg=8192.0000 max=32766 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[0, 2, 1, 12, 32754, 32766, 1, 0] avg=8192.0000 max=32766 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[0, 2, 1, 12, 32754, 32766, 1, 0] avg=8192.0000 max=32766 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[7945, 8276, 8274, 8275, 7946, 8272, 8275, 8273] avg=8192.0000 max=8276 max_rank=1 min=7945 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97969 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[0, 2, 1, 12, 32754, 32766, 1, 0] avg=8192.0000 max=32766 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[0, 2, 1, 12, 32754, 32766, 1, 0] avg=8192.0000 max=32766 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[8276, 8274, 7944, 7948, 8274, 8275, 8273, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[8276, 8274, 7944, 7948, 8274, 8275, 8273, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[8276, 8274, 7944, 7948, 8274, 8275, 8273, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[0, 2, 1, 12, 32754, 32766, 1, 0] avg=8192.0000 max=32766 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[32762, 4, 1, 0, 32766, 0, 1, 2] avg=8192.0000 max=32766 max_rank=4 min=0 min_rank=3 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[8276, 8274, 7944, 7948, 8274, 8275, 8273, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[8276, 8274, 7944, 7948, 8274, 8275, 8273, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[0, 2, 1, 12, 32754, 32766, 1, 0] avg=8192.0000 max=32766 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[8272, 7948, 7944, 8276, 8275, 8272, 8275, 8274] avg=8192.0000 max=8276 max_rank=3 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[8276, 8274, 7944, 7948, 8274, 8275, 8273, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[32762, 4, 1, 0, 32766, 0, 1, 2] avg=8192.0000 max=32766 max_rank=4 min=0 min_rank=3 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[32762, 4, 1, 0, 32766, 0, 1, 2] avg=8192.0000 max=32766 max_rank=4 min=0 min_rank=3 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[32762, 4, 1, 0, 32766, 0, 1, 2] avg=8192.0000 max=32766 max_rank=4 min=0 min_rank=3 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[8276, 8274, 7944, 7948, 8274, 8275, 8273, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97955 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[32762, 4, 1, 0, 32766, 0, 1, 2] avg=8192.0000 max=32766 max_rank=4 min=0 min_rank=3 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[32762, 4, 1, 0, 32766, 0, 1, 2] avg=8192.0000 max=32766 max_rank=4 min=0 min_rank=3 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[32766, 0, 1, 1, 2, 32766, 0, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[8272, 7948, 7944, 8276, 8275, 8272, 8275, 8274] avg=8192.0000 max=8276 max_rank=3 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[8272, 7948, 7944, 8276, 8275, 8272, 8275, 8274] avg=8192.0000 max=8276 max_rank=3 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[8272, 7948, 7944, 8276, 8275, 8272, 8275, 8274] avg=8192.0000 max=8276 max_rank=3 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[32762, 4, 1, 0, 32766, 0, 1, 2] avg=8192.0000 max=32766 max_rank=4 min=0 min_rank=3 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[8272, 7948, 7944, 8276, 8275, 8272, 8275, 8274] avg=8192.0000 max=8276 max_rank=3 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[8272, 7948, 7944, 8276, 8275, 8272, 8275, 8274] avg=8192.0000 max=8276 max_rank=3 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[32762, 4, 1, 0, 32766, 0, 1, 2] avg=8192.0000 max=32766 max_rank=4 min=0 min_rank=3 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[8275, 8272, 8275, 8273, 7947, 8275, 8275, 7944] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[8272, 7948, 7944, 8276, 8275, 8272, 8275, 8274] avg=8192.0000 max=8276 max_rank=3 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[32766, 0, 1, 1, 2, 32766, 0, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[32766, 0, 1, 1, 2, 32766, 0, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[32766, 0, 1, 1, 2, 32766, 0, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[8272, 7948, 7944, 8276, 8275, 8272, 8275, 8274] avg=8192.0000 max=8276 max_rank=3 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97964 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[32766, 0, 1, 1, 2, 32766, 0, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[32766, 0, 1, 1, 2, 32766, 0, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[2, 32766, 0, 0, 1, 1, 0, 32766] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[8275, 8272, 8275, 8273, 7947, 8275, 8275, 7944] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[8275, 8272, 8275, 8273, 7947, 8275, 8275, 7944] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[8275, 8272, 8275, 8273, 7947, 8275, 8275, 7944] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[32766, 0, 1, 1, 2, 32766, 0, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[8275, 8272, 8275, 8273, 7947, 8275, 8275, 7944] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[8275, 8272, 8275, 8273, 7947, 8275, 8275, 7944] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[32766, 0, 1, 1, 2, 32766, 0, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[7950, 8276, 8275, 8272, 8273, 8273, 7943, 8274] avg=8192.0000 max=8276 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[8275, 8272, 8275, 8273, 7947, 8275, 8275, 7944] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[2, 32766, 0, 0, 1, 1, 0, 32766] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[2, 32766, 0, 0, 1, 1, 0, 32766] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[2, 32766, 0, 0, 1, 1, 0, 32766] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[8275, 8272, 8275, 8273, 7947, 8275, 8275, 7944] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[2, 32766, 0, 0, 1, 1, 0, 32766] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[2, 32766, 0, 0, 1, 1, 0, 32766] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[32766, 0, 2, 0, 0, 32766, 2, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[7950, 8276, 8275, 8272, 8273, 8273, 7943, 8274] avg=8192.0000 max=8276 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[7950, 8276, 8275, 8272, 8273, 8273, 7943, 8274] avg=8192.0000 max=8276 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[7950, 8276, 8275, 8272, 8273, 8273, 7943, 8274] avg=8192.0000 max=8276 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[7950, 8276, 8275, 8272, 8273, 8273, 7943, 8274] avg=8192.0000 max=8276 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[2, 32766, 0, 0, 1, 1, 0, 32766] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[7950, 8276, 8275, 8272, 8273, 8273, 7943, 8274] avg=8192.0000 max=8276 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[2, 32766, 0, 0, 1, 1, 0, 32766] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[8276, 8274, 8273, 8272, 7944, 8276, 7949, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[7950, 8276, 8275, 8272, 8273, 8273, 7943, 8274] avg=8192.0000 max=8276 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[32766, 0, 2, 0, 0, 32766, 2, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[7950, 8276, 8275, 8272, 8273, 8273, 7943, 8274] avg=8192.0000 max=8276 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[32766, 0, 2, 0, 0, 32766, 2, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[32766, 0, 2, 0, 0, 32766, 2, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[32766, 0, 2, 0, 0, 32766, 2, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[32766, 0, 2, 0, 0, 32766, 2, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[2, 32766, 0, 1, 32766, 1, 0, 0] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[8276, 8274, 8273, 8272, 7944, 8276, 7949, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[8276, 8274, 8273, 8272, 7944, 8276, 7949, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[8276, 8274, 8273, 8272, 7944, 8276, 7949, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[8276, 8274, 8273, 8272, 7944, 8276, 7949, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[8276, 8274, 8273, 8272, 7944, 8276, 7949, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[32766, 0, 2, 0, 0, 32766, 2, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[32766, 0, 2, 0, 0, 32766, 2, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[7950, 8272, 8275, 8273, 8274, 8273, 8275, 7944] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[2, 32766, 0, 1, 32766, 1, 0, 0] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[2, 32766, 0, 1, 32766, 1, 0, 0] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[8276, 8274, 8273, 8272, 7944, 8276, 7949, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[2, 32766, 0, 1, 32766, 1, 0, 0] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[8276, 8274, 8273, 8272, 7944, 8276, 7949, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[2, 0, 2, 0, 0, 32766, 32766, 0] avg=8192.0000 max=32766 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[7950, 8272, 8275, 8273, 8274, 8273, 8275, 7944] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[2, 32766, 0, 1, 32766, 1, 0, 0] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[2, 32766, 0, 1, 32766, 1, 0, 0] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[7950, 8272, 8275, 8273, 8274, 8273, 8275, 7944] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[7950, 8272, 8275, 8273, 8274, 8273, 8275, 7944] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[8274, 8276, 7949, 8274, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[7950, 8272, 8275, 8273, 8274, 8273, 8275, 7944] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[7950, 8272, 8275, 8273, 8274, 8273, 8275, 7944] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[2, 32766, 0, 1, 32766, 1, 0, 0] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[2, 32766, 0, 1, 32766, 1, 0, 0] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[2, 0, 2, 0, 0, 32766, 32766, 0] avg=8192.0000 max=32766 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[2, 0, 2, 0, 0, 32766, 32766, 0] avg=8192.0000 max=32766 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[7950, 8272, 8275, 8273, 8274, 8273, 8275, 7944] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[2, 0, 2, 0, 0, 32766, 32766, 0] avg=8192.0000 max=32766 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[2, 0, 32766, 2, 0, 32764, 0, 2] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[8274, 8276, 7949, 8274, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[2, 0, 2, 0, 0, 32766, 32766, 0] avg=8192.0000 max=32766 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[2, 0, 2, 0, 0, 32766, 32766, 0] avg=8192.0000 max=32766 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[7950, 8272, 8275, 8273, 8274, 8273, 8275, 7944] avg=8192.0000 max=8275 max_rank=2 min=7944 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[8274, 8276, 7949, 8274, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[8274, 8276, 8275, 8274, 8272, 8274, 7943, 7948] avg=8192.0000 max=8276 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[8274, 8276, 7949, 8274, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[8274, 8276, 7949, 8274, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[8274, 8276, 7949, 8274, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[2, 0, 2, 0, 0, 32766, 32766, 0] avg=8192.0000 max=32766 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[2, 0, 32766, 2, 0, 32764, 0, 2] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[2, 0, 32766, 2, 0, 32764, 0, 2] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[2, 0, 2, 0, 0, 32766, 32766, 0] avg=8192.0000 max=32766 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[32766, 0, 2, 2, 32766, 0, 0, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[8274, 8276, 7949, 8274, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[2, 0, 32766, 2, 0, 32764, 0, 2] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[8274, 8276, 8275, 8274, 8272, 8274, 7943, 7948] avg=8192.0000 max=8276 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[2, 0, 32766, 2, 0, 32764, 0, 2] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[2, 0, 32766, 2, 0, 32764, 0, 2] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[8274, 8276, 8275, 8274, 8272, 8274, 7943, 7948] avg=8192.0000 max=8276 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[8274, 8276, 7949, 8274, 7944, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[8276, 8275, 8273, 7947, 8275, 8272, 7943, 8275] avg=8192.0000 max=8276 max_rank=0 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[8274, 8276, 8275, 8274, 8272, 8274, 7943, 7948] avg=8192.0000 max=8276 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[8274, 8276, 8275, 8274, 8272, 8274, 7943, 7948] avg=8192.0000 max=8276 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[8274, 8276, 8275, 8274, 8272, 8274, 7943, 7948] avg=8192.0000 max=8276 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[2, 0, 32766, 2, 0, 32764, 0, 2] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[32766, 0, 2, 2, 32766, 0, 0, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[32766, 0, 2, 2, 32766, 0, 0, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[2, 0, 32766, 2, 0, 32764, 0, 2] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[0, 2, 0, 32762, 1, 2, 3, 32766] avg=8192.0000 max=32766 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[8274, 8276, 8275, 8274, 8272, 8274, 7943, 7948] avg=8192.0000 max=8276 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[8276, 8275, 8273, 7947, 8275, 8272, 7943, 8275] avg=8192.0000 max=8276 max_rank=0 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[32766, 0, 2, 2, 32766, 0, 0, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[32766, 0, 2, 2, 32766, 0, 0, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[32766, 0, 2, 2, 32766, 0, 0, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[8276, 8275, 8273, 7947, 8275, 8272, 7943, 8275] avg=8192.0000 max=8276 max_rank=0 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[8274, 8276, 8275, 8274, 8272, 8274, 7943, 7948] avg=8192.0000 max=8276 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97966 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[8275, 8273, 8275, 8275, 7945, 8273, 7948, 8272] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[8276, 8275, 8273, 7947, 8275, 8272, 7943, 8275] avg=8192.0000 max=8276 max_rank=0 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[8276, 8275, 8273, 7947, 8275, 8272, 7943, 8275] avg=8192.0000 max=8276 max_rank=0 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[0, 2, 0, 32762, 1, 2, 3, 32766] avg=8192.0000 max=32766 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[32766, 0, 2, 2, 32766, 0, 0, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[8276, 8275, 8273, 7947, 8275, 8272, 7943, 8275] avg=8192.0000 max=8276 max_rank=0 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[0, 2, 0, 32762, 1, 2, 3, 32766] avg=8192.0000 max=32766 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[2, 32766, 0, 0, 0, 1, 1, 32766] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[32766, 0, 2, 2, 32766, 0, 0, 0] avg=8192.0000 max=32766 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[8275, 8273, 8275, 8275, 7945, 8273, 7948, 8272] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[8276, 8275, 8273, 7947, 8275, 8272, 7943, 8275] avg=8192.0000 max=8276 max_rank=0 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[0, 2, 0, 32762, 1, 2, 3, 32766] avg=8192.0000 max=32766 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[0, 2, 0, 32762, 1, 2, 3, 32766] avg=8192.0000 max=32766 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[0, 2, 0, 32762, 1, 2, 3, 32766] avg=8192.0000 max=32766 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[8275, 8273, 8275, 8275, 7945, 8273, 7948, 8272] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[7947, 8275, 8275, 8272, 7944, 8273, 8275, 8275] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[8276, 8275, 8273, 7947, 8275, 8272, 7943, 8275] avg=8192.0000 max=8276 max_rank=0 min=7943 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24490 max_load_reduction_pct=0.747421 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[8275, 8273, 8275, 8275, 7945, 8273, 7948, 8272] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[8275, 8273, 8275, 8275, 7945, 8273, 7948, 8272] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[8275, 8273, 8275, 8275, 7945, 8273, 7948, 8272] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[2, 32766, 0, 0, 0, 1, 1, 32766] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[0, 2, 0, 32762, 1, 2, 3, 32766] avg=8192.0000 max=32766 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[2, 32766, 0, 0, 0, 1, 1, 32766] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[0, 2, 0, 32762, 1, 2, 3, 32766] avg=8192.0000 max=32766 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[7947, 8275, 8275, 8272, 7944, 8273, 8275, 8275] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[8275, 8273, 8275, 8275, 7945, 8273, 7948, 8272] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[2, 32766, 0, 0, 0, 1, 1, 32766] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[2, 32766, 0, 0, 0, 1, 1, 32766] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[2, 32766, 0, 0, 0, 1, 1, 32766] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[7947, 8275, 8275, 8272, 7944, 8273, 8275, 8275] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[8275, 8273, 8275, 8275, 7945, 8273, 7948, 8272] avg=8192.0000 max=8275 max_rank=0 min=7945 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[7947, 8275, 8275, 8272, 7944, 8273, 8275, 8275] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[7947, 8275, 8275, 8272, 7944, 8273, 8275, 8275] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[7947, 8275, 8275, 8272, 7944, 8273, 8275, 8275] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[2, 32766, 0, 0, 0, 1, 1, 32766] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[2, 32766, 0, 0, 0, 1, 1, 32766] avg=8192.0000 max=32766 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[7947, 8275, 8275, 8272, 7944, 8273, 8275, 8275] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[7947, 8275, 8275, 8272, 7944, 8273, 8275, 8275] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97970 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=32 total_expected_recv=256077 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=215680 peak_demand=8276 required_capacity=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=32 total_expected_recv=262333 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=209280 peak_demand=9758 required_capacity=1.191162 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=32 total_expected_recv=266369 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=205952 peak_demand=9759 required_capacity=1.191284 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=32 total_expected_recv=261394 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=210560 peak_demand=8961 required_capacity=1.093872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=32 total_expected_recv=265367 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=206848 peak_demand=9757 required_capacity=1.191040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=32 total_expected_recv=262391 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=209280 peak_demand=9758 required_capacity=1.191162 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=32 total_expected_recv=258490 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=212224 peak_demand=9032 required_capacity=1.102539 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=32 total_expected_recv=264731 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=206464 peak_demand=9758 required_capacity=1.191162 + [2026-08-30 05:09:18] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 912.3 | throughput per GPU (TFLOP/s/GPU): 365.4 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.631618E+00 | loss scale: 1.0 | grad norm: 6.490 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 14208 | min safe capacity: 1.7344 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=14848 expected_recv=3904 expected_overflow=0 exact_recv_per_rank=[11717, 11, 6191, 11717, 11717, 6439, 9394, 8350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=14848 expected_recv=21619 expected_overflow=0 exact_recv_per_rank=[11717, 11, 6191, 11717, 11717, 6439, 9394, 8350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=14848 expected_recv=13565 expected_overflow=0 exact_recv_per_rank=[11717, 11, 6191, 11717, 11717, 6439, 9394, 8350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=14848 expected_recv=21618 expected_overflow=0 exact_recv_per_rank=[11717, 11, 6191, 11717, 11717, 6439, 9394, 8350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=14848 expected_recv=11630 expected_overflow=0 exact_recv_per_rank=[11717, 11, 6191, 11717, 11717, 6439, 9394, 8350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=14848 expected_recv=17822 expected_overflow=0 exact_recv_per_rank=[11717, 11, 6191, 11717, 11717, 6439, 9394, 8350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=14848 expected_recv=3904 actual_valid_recv=4224 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=14848 expected_recv=21619 expected_overflow=0 exact_recv_per_rank=[11717, 11, 6191, 11717, 11717, 6439, 9394, 8350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=14848 expected_recv=19295 expected_overflow=0 exact_recv_per_rank=[11717, 11, 6191, 11717, 11717, 6439, 9394, 8350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=14848 expected_recv=21619 actual_valid_recv=21760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=14848 expected_recv=13565 actual_valid_recv=13696 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=14848 expected_recv=21618 actual_valid_recv=21760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=4224 slack_slots=25472 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=14848 expected_recv=11630 actual_valid_recv=11776 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=14848 expected_recv=17822 actual_valid_recv=18176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=21760 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=13696 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=21760 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=14848 expected_recv=21619 actual_valid_recv=21760 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=11776 slack_slots=17920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=18176 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=14848 expected_recv=19295 actual_valid_recv=19456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=21760 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=19456 slack_slots=10240 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8272, 8275, 7944, 7947, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=14848 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8272, 8275, 7944, 7947, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8272, 8275, 7944, 7947, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8272, 8275, 7944, 7947, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=14848 expected_recv=15896 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8272, 8275, 7944, 7947, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=14848 expected_recv=16218 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8272, 8275, 7944, 7947, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=14848 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8272, 8275, 7944, 7947, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=14848 expected_recv=15896 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8272, 8275, 7944, 7947, 8274, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=14848 expected_recv=16217 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 7946, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=14848 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 7946, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 7946, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=14848 expected_recv=15888 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 7946, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 7946, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 7946, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=14848 expected_recv=15892 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 7946, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 7946, 8274, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=14848 expected_recv=15892 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8274, 8272, 7940, 8275, 7953, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8272, 7940, 8275, 7953, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=14848 expected_recv=15877 expected_overflow=0 exact_recv_per_rank=[8274, 8272, 7940, 8275, 7953, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=14848 expected_recv=15877 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8272, 7940, 8275, 7953, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8272, 7940, 8275, 7953, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8272, 7940, 8275, 7953, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=14848 expected_recv=15908 expected_overflow=0 exact_recv_per_rank=[8274, 8272, 7940, 8275, 7953, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8272, 7940, 8275, 7953, 8275, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=14848 expected_recv=15908 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 8276, 7944, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 8276, 7944, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 8276, 7944, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=14848 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 8276, 7944, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 8276, 7944, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 8276, 7944, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=14848 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 8276, 7944, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=14848 expected_recv=15888 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 8276, 7944, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=14848 expected_recv=15888 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 7946, 8275, 8272, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=14848 expected_recv=15892 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 7946, 8275, 8272, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=14848 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 7946, 8275, 8272, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=14848 expected_recv=15892 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 7946, 8275, 8272, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=14848 expected_recv=15888 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 7946, 8275, 8272, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 7946, 8275, 8272, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 7946, 8275, 8272, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 7946, 8275, 8272, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=14848 expected_recv=27518 expected_overflow=4068 exact_recv_per_rank=[13852, 18916, 13852, 0, 18916, 0, 0, 0] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=14848 expected_recv=8604 expected_overflow=0 exact_recv_per_rank=[13852, 18916, 13852, 0, 18916, 0, 0, 0] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=14848 expected_recv=22456 expected_overflow=0 exact_recv_per_rank=[13852, 18916, 13852, 0, 18916, 0, 0, 0] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=14848 expected_recv=27518 actual_valid_recv=23552 observed_loss=4068 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=14848 expected_recv=7417 expected_overflow=0 exact_recv_per_rank=[13852, 18916, 13852, 0, 18916, 0, 0, 0] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=14848 expected_recv=8604 actual_valid_recv=8704 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=23552 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=14848 expected_recv=22456 actual_valid_recv=22656 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=14848 expected_recv=7417 actual_valid_recv=7424 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=8704 slack_slots=20992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=22656 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=14848 expected_recv=6960 expected_overflow=0 exact_recv_per_rank=[13852, 18916, 13852, 0, 18916, 0, 0, 0] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=7424 slack_slots=22272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=14848 expected_recv=6960 actual_valid_recv=7040 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=14848 expected_recv=22455 expected_overflow=0 exact_recv_per_rank=[13852, 18916, 13852, 0, 18916, 0, 0, 0] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=14848 expected_recv=27516 expected_overflow=4068 exact_recv_per_rank=[13852, 18916, 13852, 0, 18916, 0, 0, 0] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=7040 slack_slots=22656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=14848 expected_recv=22455 actual_valid_recv=22656 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=14848 expected_recv=8146 expected_overflow=0 exact_recv_per_rank=[13852, 18916, 13852, 0, 18916, 0, 0, 0] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=14848 expected_recv=27516 actual_valid_recv=23552 observed_loss=4068 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=22656 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=14848 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=14848 expected_recv=8146 actual_valid_recv=8192 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=14848 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=23552 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=14848 expected_recv=15888 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=8192 slack_slots=21504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=14848 expected_recv=15888 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=14848 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=14848 expected_recv=15888 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 8276, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 8276, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=14848 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 8276, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=14848 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 8276, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 8276, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=14848 expected_recv=15888 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=14848 expected_recv=15888 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=14848 expected_recv=15890 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 8276, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=14848 expected_recv=15890 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 8276, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 8272, 7944, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=14848 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 8272, 7944, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=14848 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 8272, 7944, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7945, 8276, 8276, 7944, 8276, 8272, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=14848 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 8272, 7944, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=14848 expected_recv=16217 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=14848 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 8272, 7944, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=14848 expected_recv=16216 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=14848 expected_recv=16220 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=14848 expected_recv=16220 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 8272, 7944, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 8272, 7944, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=14848 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8275, 8276, 8272, 8272, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8275, 8276, 8272, 8272, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=14848 expected_recv=15889 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8275, 8276, 8272, 8272, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7945, 8276, 8272, 7944, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8275, 8276, 8272, 8272, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=14848 expected_recv=15888 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8275, 8276, 8272, 8272, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=14848 expected_recv=15889 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8275, 8276, 8272, 8272, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8275, 8276, 8272, 8272, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8276, 8272, 7946, 8274, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8276, 8272, 7946, 8274, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8276, 8272, 7946, 8274, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8275, 8276, 8272, 8272, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8276, 8272, 7946, 8274, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=14848 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8276, 8272, 7946, 8274, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=14848 expected_recv=16216 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8276, 8272, 7946, 8274, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=14848 expected_recv=15892 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8276, 8272, 7946, 8274, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8276, 8272, 7945, 7944, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=14848 expected_recv=15892 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8276, 8272, 7945, 7944, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8276, 8272, 7945, 7944, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=14848 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8276, 8272, 7946, 8274, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8276, 8272, 7945, 7944, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=14848 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8276, 8272, 7945, 7944, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=14848 expected_recv=16218 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=14848 expected_recv=16216 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8276, 8272, 7945, 7944, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=14848 expected_recv=15891 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8276, 8272, 7945, 7944, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=14848 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8276, 8276, 7944, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=14848 expected_recv=15891 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=14848 expected_recv=15888 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8276, 8276, 7944, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8276, 8276, 7944, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=14848 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8276, 8276, 7944, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8276, 8276, 7944, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8274, 8276, 8272, 7945, 7944, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=14848 expected_recv=15888 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8276, 8276, 7944, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8276, 8276, 7944, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8275, 8274, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=14848 expected_recv=15892 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8275, 8274, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8275, 8274, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8275, 8274, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=14848 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8275, 8274, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=14848 expected_recv=15892 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 7944, 8276, 8276, 7944, 8272, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=14848 expected_recv=15888 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8275, 8274, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8275, 8274, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[11717, 11, 6191, 21111, 20065, 6439, 0, 2] avg=8192.0000 max=21111 max_rank=3 min=0 min_rank=6 imbalance_ratio=2.577026 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[11717, 11, 6191, 21111, 20065, 6439, 0, 2] avg=8192.0000 max=21111 max_rank=3 min=0 min_rank=6 imbalance_ratio=2.577026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[11717, 11, 6191, 21111, 20065, 6439, 0, 2] avg=8192.0000 max=21111 max_rank=3 min=0 min_rank=6 imbalance_ratio=2.577026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[11717, 11, 6191, 21111, 20065, 6439, 0, 2] avg=8192.0000 max=21111 max_rank=3 min=0 min_rank=6 imbalance_ratio=2.577026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[11717, 11, 6191, 21111, 20065, 6439, 0, 2] avg=8192.0000 max=21111 max_rank=3 min=0 min_rank=6 imbalance_ratio=2.577026 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8272, 7944, 8275, 8274, 8273, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[11717, 11, 6191, 11717, 11717, 6439, 9394, 8350] avg=8192.0000 max=11717 max_rank=0 min=11 min_rank=1 imbalance_ratio=1.430298 max_load_reduction=9394 max_load_reduction_pct=0.444981 moved_to_replicas=41002 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[11717, 11, 6191, 11717, 11717, 6439, 9394, 8350] avg=8192.0000 max=11717 max_rank=0 min=11 min_rank=1 imbalance_ratio=1.430298 max_load_reduction=9394 max_load_reduction_pct=0.444981 moved_to_replicas=41002 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[11717, 11, 6191, 11717, 11717, 6439, 9394, 8350] avg=8192.0000 max=11717 max_rank=0 min=11 min_rank=1 imbalance_ratio=1.430298 max_load_reduction=9394 max_load_reduction_pct=0.444981 moved_to_replicas=41002 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[11717, 11, 6191, 11717, 11717, 6439, 9394, 8350] avg=8192.0000 max=11717 max_rank=0 min=11 min_rank=1 imbalance_ratio=1.430298 max_load_reduction=9394 max_load_reduction_pct=0.444981 moved_to_replicas=41002 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[11717, 11, 6191, 21111, 20065, 6439, 0, 2] avg=8192.0000 max=21111 max_rank=3 min=0 min_rank=6 imbalance_ratio=2.577026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[11717, 11, 6191, 11717, 11717, 6439, 9394, 8350] avg=8192.0000 max=11717 max_rank=0 min=11 min_rank=1 imbalance_ratio=1.430298 max_load_reduction=9394 max_load_reduction_pct=0.444981 moved_to_replicas=41002 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[11717, 11, 6191, 21111, 20065, 6439, 0, 2] avg=8192.0000 max=21111 max_rank=3 min=0 min_rank=6 imbalance_ratio=2.577026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[0, 32713, 32766, 0, 0, 44, 1, 12] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[11717, 11, 6191, 11717, 11717, 6439, 9394, 8350] avg=8192.0000 max=11717 max_rank=0 min=11 min_rank=1 imbalance_ratio=1.430298 max_load_reduction=9394 max_load_reduction_pct=0.444981 moved_to_replicas=41002 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[11717, 11, 6191, 11717, 11717, 6439, 9394, 8350] avg=8192.0000 max=11717 max_rank=0 min=11 min_rank=1 imbalance_ratio=1.430298 max_load_reduction=9394 max_load_reduction_pct=0.444981 moved_to_replicas=41002 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[0, 32713, 32766, 0, 0, 44, 1, 12] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[0, 32713, 32766, 0, 0, 44, 1, 12] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[11717, 11, 6191, 21111, 20065, 6439, 0, 2] avg=8192.0000 max=21111 max_rank=3 min=0 min_rank=6 imbalance_ratio=2.577026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[0, 32713, 32766, 0, 0, 44, 1, 12] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[0, 32713, 32766, 0, 0, 44, 1, 12] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[8275, 8274, 8272, 8275, 7944, 7947, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[8275, 8274, 8272, 8275, 7944, 7947, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[8275, 8274, 8272, 8275, 7944, 7947, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[0, 32713, 32766, 0, 0, 44, 1, 12] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[8275, 8274, 8272, 8275, 7944, 7947, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[0, 32713, 32766, 0, 0, 44, 1, 12] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[11717, 11, 6191, 11717, 11717, 6439, 9394, 8350] avg=8192.0000 max=11717 max_rank=0 min=11 min_rank=1 imbalance_ratio=1.430298 max_load_reduction=9394 max_load_reduction_pct=0.444981 moved_to_replicas=41002 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[8275, 8274, 8272, 8275, 7944, 7947, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[0, 0, 32768, 0, 6951, 25817, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[0, 0, 32768, 0, 6951, 25817, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[8275, 8274, 8272, 8275, 7944, 7947, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[8275, 8274, 8272, 8275, 7944, 7947, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[0, 0, 32768, 0, 6951, 25817, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[0, 0, 32768, 0, 6951, 25817, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 7946, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=83291 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[0, 0, 32768, 0, 6951, 25817, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[0, 32713, 32766, 0, 0, 44, 1, 12] avg=8192.0000 max=32766 max_rank=2 min=0 min_rank=0 imbalance_ratio=3.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 7946, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=83291 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 7946, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=83291 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 7946, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=83291 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[0, 0, 32768, 0, 6951, 25817, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[0, 0, 32768, 0, 6951, 25817, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 7946, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=83291 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[8275, 8274, 8272, 8275, 7944, 7947, 8274, 8275] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24491 max_load_reduction_pct=0.747452 moved_to_replicas=97873 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[0, 0, 0, 32752, 17, 32761, 6, 0] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[0, 0, 0, 32752, 17, 32761, 6, 0] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 7946, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=83291 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[0, 0, 0, 32752, 17, 32761, 6, 0] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 7946, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=83291 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[0, 0, 0, 32752, 17, 32761, 6, 0] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[8274, 8272, 7940, 8275, 7953, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=3 min=7940 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97914 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[8274, 8272, 7940, 8275, 7953, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=3 min=7940 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97914 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[0, 0, 0, 32752, 17, 32761, 6, 0] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[0, 0, 32768, 0, 6951, 25817, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[8274, 8272, 7940, 8275, 7953, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=3 min=7940 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97914 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[8274, 8272, 7940, 8275, 7953, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=3 min=7940 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97914 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[0, 0, 0, 32752, 17, 32761, 6, 0] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[0, 0, 0, 32752, 17, 32761, 6, 0] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 7946, 8274, 8274, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=83291 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[8274, 8272, 7940, 8275, 7953, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=3 min=7940 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97914 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[0, 0, 32768, 32768, 0, 0, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[0, 0, 32768, 32768, 0, 0, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[0, 0, 32768, 32768, 0, 0, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[8274, 8272, 7940, 8275, 7953, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=3 min=7940 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97914 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[8274, 8272, 7940, 8275, 7953, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=3 min=7940 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97914 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[8276, 8276, 8272, 8276, 7944, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[8276, 8276, 8272, 8276, 7944, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[0, 0, 32768, 32768, 0, 0, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[0, 0, 0, 32752, 17, 32761, 6, 0] avg=8192.0000 max=32761 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[0, 0, 32768, 32768, 0, 0, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[8276, 8276, 8272, 8276, 7944, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[0, 0, 32768, 32768, 0, 0, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[0, 0, 32768, 32768, 0, 0, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[8276, 8276, 8272, 8276, 7944, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[0, 0, 0, 18, 32750, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[8274, 8272, 7940, 8275, 7953, 8275, 8273, 8274] avg=8192.0000 max=8275 max_rank=3 min=7940 min_rank=2 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=97914 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[0, 0, 0, 18, 32750, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[8276, 8276, 8272, 8276, 7944, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[0, 0, 0, 18, 32750, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[8276, 8276, 8272, 8276, 7944, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[8276, 8276, 8272, 8276, 7944, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[8276, 8276, 7944, 7946, 8275, 8272, 8273, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97937 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[8276, 8276, 7944, 7946, 8275, 8272, 8273, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97937 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[0, 0, 0, 18, 32750, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[0, 0, 32768, 32768, 0, 0, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[8276, 8276, 7944, 7946, 8275, 8272, 8273, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97937 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[0, 0, 0, 18, 32750, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[0, 0, 0, 18, 32750, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[0, 0, 0, 18, 32750, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[8276, 8276, 7944, 7946, 8275, 8272, 8273, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97937 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[13852, 18916, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[8276, 8276, 8272, 8276, 7944, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[13852, 18916, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[8276, 8276, 7944, 7946, 8275, 8272, 8273, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97937 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[8276, 8276, 7944, 7946, 8275, 8272, 8273, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97937 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[13852, 18916, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[8276, 8276, 7944, 7946, 8275, 8272, 8273, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97937 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[13852, 18916, 13852, 0, 18916, 0, 0, 0] avg=8192.0000 max=18916 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.309082 max_load_reduction=13852 max_load_reduction_pct=0.422729 moved_to_replicas=53582 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[13852, 18916, 13852, 0, 18916, 0, 0, 0] avg=8192.0000 max=18916 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.309082 max_load_reduction=13852 max_load_reduction_pct=0.422729 moved_to_replicas=53582 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[13852, 18916, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[0, 0, 0, 18, 32750, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[13852, 18916, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[13852, 18916, 13852, 0, 18916, 0, 0, 0] avg=8192.0000 max=18916 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.309082 max_load_reduction=13852 max_load_reduction_pct=0.422729 moved_to_replicas=53582 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[13852, 18916, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[13852, 18916, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[32768, 0, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=0 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[13852, 18916, 13852, 0, 18916, 0, 0, 0] avg=8192.0000 max=18916 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.309082 max_load_reduction=13852 max_load_reduction_pct=0.422729 moved_to_replicas=53582 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[32768, 0, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=0 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[8276, 8276, 7944, 7946, 8275, 8272, 8273, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97937 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[13852, 18916, 13852, 0, 18916, 0, 0, 0] avg=8192.0000 max=18916 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.309082 max_load_reduction=13852 max_load_reduction_pct=0.422729 moved_to_replicas=53582 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[13852, 18916, 13852, 0, 18916, 0, 0, 0] avg=8192.0000 max=18916 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.309082 max_load_reduction=13852 max_load_reduction_pct=0.422729 moved_to_replicas=53582 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[32768, 0, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=0 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[13852, 18916, 13852, 0, 18916, 0, 0, 0] avg=8192.0000 max=18916 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.309082 max_load_reduction=13852 max_load_reduction_pct=0.422729 moved_to_replicas=53582 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[32768, 0, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=0 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[13852, 18916, 0, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[32768, 0, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=0 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[32768, 0, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=0 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[32768, 0, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=0 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[0, 32768, 0, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[0, 32768, 0, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[13852, 18916, 13852, 0, 18916, 0, 0, 0] avg=8192.0000 max=18916 max_rank=1 min=0 min_rank=3 imbalance_ratio=2.309082 max_load_reduction=13852 max_load_reduction_pct=0.422729 moved_to_replicas=53582 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[0, 32768, 0, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[0, 32768, 0, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[0, 32768, 0, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[0, 32768, 0, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[32768, 0, 0, 0, 0, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=0 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[0, 32768, 0, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[1, 0, 0, 0, 32767, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[1, 0, 0, 0, 32767, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[1, 0, 0, 0, 32767, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 8272, 7944] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[7945, 8276, 8276, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[7945, 8276, 8276, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[7945, 8276, 8276, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[1, 0, 0, 0, 32767, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[1, 0, 0, 0, 32767, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[1, 0, 0, 0, 32767, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[0, 32768, 0, 0, 0, 0, 0, 32768] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[0, 32767, 1, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[1, 0, 0, 0, 32767, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[0, 32767, 1, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[7945, 8276, 8276, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[7945, 8276, 8276, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[0, 32767, 1, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[7945, 8276, 8276, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8276, 8276, 7944, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 8272, 7944, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97975 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 8272, 7944, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97975 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[7945, 8276, 8276, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 8272, 7944, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97975 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[0, 32767, 1, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[0, 32767, 1, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[0, 32767, 1, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[1, 0, 0, 0, 32767, 32768, 0, 0] avg=8192.0000 max=32768 max_rank=5 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[0, 0, 0, 0, 0, 32767, 32768, 1] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[0, 0, 0, 0, 0, 32767, 32768, 1] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[0, 32767, 1, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 8272, 7944, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97975 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 8272, 7944, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97975 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[0, 0, 0, 0, 0, 32767, 32768, 1] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 8272, 7944, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97975 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[7945, 8276, 8276, 7944, 8276, 8272, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[8276, 8276, 7944, 8275, 8276, 8272, 8272, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97975 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[8276, 8276, 7944, 8275, 8276, 8272, 8272, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97975 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 8272, 7944, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97975 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[8276, 8276, 7944, 8275, 8276, 8272, 8272, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97975 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[0, 0, 0, 0, 0, 32767, 32768, 1] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[0, 0, 0, 0, 0, 32767, 32768, 1] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[0, 0, 0, 0, 0, 32767, 32768, 1] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[1, 0, 32768, 0, 7196, 25571, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[0, 32767, 1, 0, 32768, 0, 0, 0] avg=8192.0000 max=32768 max_rank=4 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[1, 0, 32768, 0, 7196, 25571, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[0, 0, 0, 0, 0, 32767, 32768, 1] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[8276, 8276, 7944, 8275, 8276, 8272, 8272, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97975 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[8276, 8276, 7944, 8275, 8276, 8272, 8272, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97975 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[1, 0, 32768, 0, 7196, 25571, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[8276, 8276, 7944, 8275, 8276, 8272, 8272, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97975 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[8274, 8276, 8276, 8272, 7946, 8274, 7944, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=83186 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[8274, 8276, 8276, 8272, 7946, 8274, 7944, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=83186 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[8276, 8276, 7945, 8276, 8272, 7944, 8275, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97975 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[8276, 8276, 7944, 8275, 8276, 8272, 8272, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97975 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[8274, 8276, 8276, 8272, 7946, 8274, 7944, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=83186 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[1, 0, 32768, 0, 7196, 25571, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[1, 0, 32768, 0, 7196, 25571, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[1, 0, 32768, 0, 7196, 25571, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[32768, 1, 0, 0, 24, 0, 32743, 0] avg=8192.0000 max=32768 max_rank=0 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[32768, 1, 0, 0, 24, 0, 32743, 0] avg=8192.0000 max=32768 max_rank=0 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[0, 0, 0, 0, 0, 32767, 32768, 1] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[1, 0, 32768, 0, 7196, 25571, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[32768, 1, 0, 0, 24, 0, 32743, 0] avg=8192.0000 max=32768 max_rank=0 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[8274, 8276, 8276, 8272, 7946, 8274, 7944, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=83186 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[8274, 8276, 8276, 8272, 7946, 8274, 7944, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=83186 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[8274, 8276, 8276, 8272, 7946, 8274, 7944, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=83186 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[8276, 8274, 8276, 8272, 7945, 7944, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[8276, 8274, 8276, 8272, 7945, 7944, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[8276, 8276, 7944, 8275, 8276, 8272, 8272, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97975 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[8274, 8276, 8276, 8272, 7946, 8274, 7944, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=83186 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[8276, 8274, 8276, 8272, 7945, 7944, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[32768, 1, 0, 0, 24, 0, 32743, 0] avg=8192.0000 max=32768 max_rank=0 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[32768, 1, 0, 0, 24, 0, 32743, 0] avg=8192.0000 max=32768 max_rank=0 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[32768, 1, 0, 0, 24, 0, 32743, 0] avg=8192.0000 max=32768 max_rank=0 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[0, 0, 0, 0, 0, 0, 32768, 32768] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[0, 0, 0, 0, 0, 0, 32768, 32768] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[32768, 1, 0, 0, 24, 0, 32743, 0] avg=8192.0000 max=32768 max_rank=0 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[1, 0, 32768, 0, 7196, 25571, 0, 0] avg=8192.0000 max=32768 max_rank=2 min=0 min_rank=1 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[0, 0, 0, 0, 0, 0, 32768, 32768] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[8276, 8274, 8276, 8272, 7945, 7944, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[8276, 8274, 8276, 8272, 7945, 7944, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[8276, 8274, 8276, 8272, 7945, 7944, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[8276, 8276, 7944, 8276, 8276, 7944, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[8276, 8276, 7944, 8276, 8276, 7944, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[8276, 8274, 8276, 8272, 7945, 7944, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[8274, 8276, 8276, 8272, 7946, 8274, 7944, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=83186 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[8276, 8276, 7944, 8276, 8276, 7944, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[0, 0, 0, 0, 0, 0, 32768, 32768] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[0, 0, 0, 0, 0, 0, 32768, 32768] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[0, 0, 0, 0, 0, 0, 32768, 32768] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[0, 32768, 0, 0, 31104, 0, 0, 1664] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[0, 32768, 0, 0, 31104, 0, 0, 1664] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[0, 0, 0, 0, 0, 0, 32768, 32768] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[0, 32768, 0, 0, 31104, 0, 0, 1664] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[32768, 1, 0, 0, 24, 0, 32743, 0] avg=8192.0000 max=32768 max_rank=0 min=0 min_rank=2 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[8276, 8276, 7944, 8276, 8276, 7944, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[8276, 8276, 7944, 8276, 8276, 7944, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[8276, 8276, 7944, 8276, 8276, 7944, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8275, 8274, 8273, 7946] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=94712 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8275, 8274, 8273, 7946] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=94712 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8275, 8274, 8273, 7946] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=94712 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[8276, 8276, 7944, 8276, 8276, 7944, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[8276, 8274, 8276, 8272, 7945, 7944, 8275, 8274] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97916 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[0, 32768, 0, 0, 31104, 0, 0, 1664] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[0, 32768, 0, 0, 31104, 0, 0, 1664] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[0, 32768, 0, 0, 31104, 0, 0, 1664] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[0, 32768, 0, 0, 31104, 0, 0, 1664] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[0, 0, 0, 0, 0, 0, 32768, 32768] avg=8192.0000 max=32768 max_rank=6 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8275, 8274, 8273, 7946] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=94712 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8275, 8274, 8273, 7946] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=94712 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8275, 8274, 8273, 7946] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=94712 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8275, 8274, 8273, 7946] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=94712 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[8276, 8276, 7944, 8276, 8276, 7944, 8272, 8272] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=97976 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[0, 32768, 0, 0, 31104, 0, 0, 1664] avg=8192.0000 max=32768 max_rank=1 min=0 min_rank=0 imbalance_ratio=4.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[8276, 8276, 8272, 7944, 8275, 8274, 8273, 7946] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=24492 max_load_reduction_pct=0.747437 moved_to_replicas=94712 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=32 total_expected_recv=253934 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=218496 peak_demand=9472 required_capacity=1.156250 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=32 total_expected_recv=264694 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=208000 peak_demand=13852 required_capacity=1.690918 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=32 total_expected_recv=248294 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=224896 peak_demand=8276 required_capacity=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=32 total_expected_recv=256628 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=215680 peak_demand=11717 required_capacity=1.430298 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=32 total_expected_recv=263139 total_dropped_tokens=4068 drop_rate_pct=1.545951 records_with_drop_pct=3.125000 peak_rank_overflow=4068 total_slack_slots=214272 peak_demand=18916 required_capacity=2.309082 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=32 total_expected_recv=277229 total_dropped_tokens=4068 drop_rate_pct=1.467379 records_with_drop_pct=3.125000 peak_rank_overflow=4068 total_slack_slots=199552 peak_demand=18916 required_capacity=2.309082 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=32 total_expected_recv=275132 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=197888 peak_demand=13852 required_capacity=1.690918 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=32 total_expected_recv=258102 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=214912 peak_demand=9901 required_capacity=1.208618 + [2026-08-30 05:09:19] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 903.6 | throughput per GPU (TFLOP/s/GPU): 369.0 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.462585E+00 | loss scale: 1.0 | grad norm: 5.569 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 14208 | min safe capacity: 1.7344 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=14848 expected_recv=22219 expected_overflow=0 exact_recv_per_rank=[7099, 6275, 8789, 8789, 8791, 8790, 8790, 8213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=14848 expected_recv=6292 expected_overflow=0 exact_recv_per_rank=[7099, 6275, 8789, 8789, 8791, 8790, 8790, 8213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=14848 expected_recv=22380 expected_overflow=0 exact_recv_per_rank=[7099, 6275, 8789, 8789, 8791, 8790, 8790, 8213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=14848 expected_recv=12603 expected_overflow=0 exact_recv_per_rank=[7099, 6275, 8789, 8789, 8791, 8790, 8790, 8213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=14848 expected_recv=21363 expected_overflow=0 exact_recv_per_rank=[7099, 6275, 8789, 8789, 8791, 8790, 8790, 8213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=14848 expected_recv=8215 expected_overflow=0 exact_recv_per_rank=[7099, 6275, 8789, 8789, 8791, 8790, 8790, 8213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=14848 expected_recv=22381 expected_overflow=0 exact_recv_per_rank=[7099, 6275, 8789, 8789, 8791, 8790, 8790, 8213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=14848 expected_recv=22219 actual_valid_recv=22272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=14848 expected_recv=6292 actual_valid_recv=6528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=14848 expected_recv=15619 expected_overflow=0 exact_recv_per_rank=[7099, 6275, 8789, 8789, 8791, 8790, 8790, 8213] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=14848 expected_recv=22380 actual_valid_recv=22528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=14848 expected_recv=12603 actual_valid_recv=12800 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=14848 expected_recv=21363 actual_valid_recv=21504 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=22272 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=14848 expected_recv=8215 actual_valid_recv=8448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=6528 slack_slots=23168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=14848 expected_recv=22381 actual_valid_recv=22528 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=22528 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=12800 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=21504 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=14848 expected_recv=15619 actual_valid_recv=15744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=8448 slack_slots=21248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=22528 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=15744 slack_slots=13952 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=14848 expected_recv=24391 expected_overflow=0 exact_recv_per_rank=[7489, 11336, 11337, 8905, 1, 11337, 11338, 3793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=14848 expected_recv=24391 expected_overflow=0 exact_recv_per_rank=[7489, 11336, 11337, 8905, 1, 11337, 11338, 3793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=14848 expected_recv=24391 actual_valid_recv=24448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=24448 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=14848 expected_recv=24391 actual_valid_recv=24448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=14848 expected_recv=24391 expected_overflow=0 exact_recv_per_rank=[7489, 11336, 11337, 8905, 1, 11337, 11338, 3793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=14848 expected_recv=11081 expected_overflow=0 exact_recv_per_rank=[7489, 11336, 11337, 8905, 1, 11337, 11338, 3793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=14848 expected_recv=8291 expected_overflow=0 exact_recv_per_rank=[7489, 11336, 11337, 8905, 1, 11337, 11338, 3793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=14848 expected_recv=14130 expected_overflow=0 exact_recv_per_rank=[7489, 11336, 11337, 8905, 1, 11337, 11338, 3793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=24448 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=14848 expected_recv=24391 actual_valid_recv=24448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=14848 expected_recv=24392 expected_overflow=0 exact_recv_per_rank=[7489, 11336, 11337, 8905, 1, 11337, 11338, 3793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=14848 expected_recv=5 expected_overflow=0 exact_recv_per_rank=[7489, 11336, 11337, 8905, 1, 11337, 11338, 3793] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=14848 expected_recv=11081 actual_valid_recv=11520 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=14848 expected_recv=8291 actual_valid_recv=8448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=24448 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=14848 expected_recv=14130 actual_valid_recv=14464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=14848 expected_recv=24392 actual_valid_recv=24448 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=14848 expected_recv=5 actual_valid_recv=256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=11520 slack_slots=18176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=8448 slack_slots=21248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8272, 8274, 7947, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=14464 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=24448 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=256 slack_slots=29440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8272, 8274, 7947, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=14848 expected_recv=15893 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8272, 8274, 7947, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=14848 expected_recv=15893 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=14848 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8272, 8274, 7947, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8272, 8274, 7947, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=14848 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8272, 8274, 7947, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8272, 8274, 7947, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8275, 8272, 8274, 7947, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8276, 8272, 7944, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=14848 expected_recv=16216 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=14848 expected_recv=16217 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8276, 8272, 7944, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=14848 expected_recv=15891 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8276, 8272, 7944, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=14848 expected_recv=15891 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8276, 8272, 7944, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8276, 8272, 7944, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8276, 8272, 7944, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8276, 8272, 7944, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8275, 8276, 8272, 7944, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8274, 8276, 8275, 8272, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=14848 expected_recv=15894 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8274, 8276, 8275, 8272, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8274, 8276, 8275, 8272, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=14848 expected_recv=15894 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8274, 8276, 8275, 8272, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8274, 8276, 8275, 8272, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8274, 8276, 8275, 8272, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=14848 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8274, 8276, 8275, 8272, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=14848 expected_recv=16216 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 7946, 8274, 8276, 8275, 8272, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8273, 8276, 8272, 7944, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8273, 8276, 8272, 7944, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8273, 8276, 8272, 7944, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=14848 expected_recv=15894 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8273, 8276, 8272, 7944, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8273, 8276, 8272, 7944, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=14848 expected_recv=15894 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=14848 expected_recv=15889 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8273, 8276, 8272, 7944, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8273, 8276, 8272, 7944, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8275, 8273, 8276, 8272, 7944, 7946] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=14848 expected_recv=16552 expected_overflow=0 exact_recv_per_rank=[7948, 8274, 8276, 8275, 8272, 7944, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7948, 8274, 8276, 8275, 8272, 7944, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=14848 expected_recv=15889 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=14848 expected_recv=15886 expected_overflow=0 exact_recv_per_rank=[7948, 8274, 8276, 8275, 8272, 7944, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=14848 expected_recv=16552 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=14848 expected_recv=15886 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7948, 8274, 8276, 8275, 8272, 7944, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7948, 8274, 8276, 8275, 8272, 7944, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7948, 8274, 8276, 8275, 8272, 7944, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[7948, 8274, 8276, 8275, 8272, 7944, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=14848 expected_recv=15898 expected_overflow=0 exact_recv_per_rank=[7948, 8274, 8276, 8275, 8272, 7944, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=14848 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8274, 8274, 7947, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8274, 8274, 7947, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=14848 expected_recv=16222 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8274, 8274, 7947, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=14848 expected_recv=15898 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=14848 expected_recv=16217 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=14848 expected_recv=16222 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8274, 8274, 7947, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8274, 8274, 7947, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=14848 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8274, 8274, 7947, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=14848 expected_recv=16221 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8274, 8274, 7947, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 7944, 8272, 8273, 7946, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8272, 8274, 8274, 7947, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=14848 expected_recv=16216 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=14848 expected_recv=15895 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 7944, 8272, 8273, 7946, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 7944, 8272, 8273, 7946, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=14848 expected_recv=16221 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=14848 expected_recv=15895 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 7944, 8272, 8273, 7946, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 7944, 8272, 8273, 7946, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 7944, 8272, 8273, 7946, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 7944, 8272, 8273, 7946, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8274, 8275, 7947, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=14848 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 7944, 8272, 8273, 7946, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8274, 8275, 7947, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8274, 8275, 7947, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=14848 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8274, 8275, 7947, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=14848 expected_recv=16220 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8274, 8275, 7947, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=14848 expected_recv=15888 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=14848 expected_recv=15896 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8274, 8275, 7947, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8274, 8275, 7947, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=14848 expected_recv=15891 expected_overflow=0 exact_recv_per_rank=[7948, 8276, 7946, 8274, 8272, 8273, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=14848 expected_recv=15896 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8276, 8274, 8275, 7947, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7948, 8276, 7946, 8274, 8272, 8273, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[7948, 8276, 7946, 8274, 8272, 8273, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=14848 expected_recv=15891 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[7948, 8276, 7946, 8274, 8272, 8273, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7948, 8276, 7946, 8274, 8272, 8273, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7948, 8276, 7946, 8274, 8272, 8273, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7948, 8276, 7946, 8274, 8272, 8273, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 8272, 7944, 8274, 7950, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=14848 expected_recv=15895 expected_overflow=0 exact_recv_per_rank=[7948, 8276, 7946, 8274, 8272, 8273, 8275, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 8272, 7944, 8274, 7950, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 8272, 7944, 8274, 7950, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 8272, 7944, 8274, 7950, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=14848 expected_recv=15895 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 8272, 7944, 8274, 7950, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=14848 expected_recv=15884 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 8272, 7944, 8274, 7950, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=14848 expected_recv=15901 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 8272, 7944, 8274, 7950, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=14848 expected_recv=15884 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=14848 expected_recv=15901 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8274, 7948, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8274, 7948, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8273, 8272, 7944, 8274, 7950, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=14848 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8274, 7948, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8274, 7948, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8274, 7948, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=14848 expected_recv=15888 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=14848 expected_recv=15896 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8274, 7948, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8274, 7948, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=14848 expected_recv=15896 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8274, 8274, 7947, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8274, 8274, 7947, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=14848 expected_recv=16216 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8274, 8274, 7947, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8273, 8276, 8275, 8274, 7948, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=14848 expected_recv=16220 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8274, 8274, 7947, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8274, 8274, 7947, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=14848 expected_recv=16216 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=14848 expected_recv=16220 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=14848 expected_recv=15894 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8274, 8274, 7947, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8274, 8274, 7947, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=14848 expected_recv=15894 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 7945, 8274, 8276, 8274, 8274, 7946, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 7945, 8274, 8276, 8274, 8274, 7946, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 7945, 8274, 8276, 8274, 8274, 7946, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8276, 8274, 8274, 7947, 8272, 8274, 7944] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=14848 expected_recv=15891 expected_overflow=0 exact_recv_per_rank=[8275, 7945, 8274, 8276, 8274, 8274, 7946, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 7945, 8274, 8276, 8274, 8274, 7946, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=14848 expected_recv=15891 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 7945, 8274, 8276, 8274, 8274, 7946, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=14848 expected_recv=15890 expected_overflow=0 exact_recv_per_rank=[8275, 7945, 8274, 8276, 8274, 8274, 7946, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=14848 expected_recv=15890 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8272, 8275, 7944, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=14848 expected_recv=15893 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8272, 8275, 7944, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=14848 expected_recv=15887 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8272, 8275, 7944, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 7945, 8274, 8276, 8274, 8274, 7946, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8272, 8275, 7944, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=14848 expected_recv=15893 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=14848 expected_recv=15887 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8272, 8275, 7944, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8272, 8275, 7944, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8272, 8275, 7944, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[6306, 28, 15036, 12763, 5610, 25793, 0, 0] avg=8192.0000 max=25793 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.148560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8276, 8276, 8275, 8272, 8275, 7944, 8273, 7945] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[6306, 28, 15036, 12763, 5610, 25793, 0, 0] avg=8192.0000 max=25793 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.148560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[6306, 28, 15036, 12763, 5610, 25793, 0, 0] avg=8192.0000 max=25793 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.148560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[6306, 28, 15036, 12763, 5610, 25793, 0, 0] avg=8192.0000 max=25793 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.148560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[6306, 28, 15036, 12763, 5610, 25793, 0, 0] avg=8192.0000 max=25793 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.148560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[7099, 6275, 8789, 8789, 8791, 8790, 8790, 8213] avg=8192.0000 max=8791 max_rank=4 min=6275 min_rank=1 imbalance_ratio=1.073120 max_load_reduction=17002 max_load_reduction_pct=0.659171 moved_to_replicas=40812 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[6306, 28, 15036, 12763, 5610, 25793, 0, 0] avg=8192.0000 max=25793 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.148560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[6306, 28, 15036, 12763, 5610, 25793, 0, 0] avg=8192.0000 max=25793 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.148560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[7099, 6275, 8789, 8789, 8791, 8790, 8790, 8213] avg=8192.0000 max=8791 max_rank=4 min=6275 min_rank=1 imbalance_ratio=1.073120 max_load_reduction=17002 max_load_reduction_pct=0.659171 moved_to_replicas=40812 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[7099, 6275, 8789, 8789, 8791, 8790, 8790, 8213] avg=8192.0000 max=8791 max_rank=4 min=6275 min_rank=1 imbalance_ratio=1.073120 max_load_reduction=17002 max_load_reduction_pct=0.659171 moved_to_replicas=40812 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[7099, 6275, 8789, 8789, 8791, 8790, 8790, 8213] avg=8192.0000 max=8791 max_rank=4 min=6275 min_rank=1 imbalance_ratio=1.073120 max_load_reduction=17002 max_load_reduction_pct=0.659171 moved_to_replicas=40812 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[7099, 6275, 8789, 8789, 8791, 8790, 8790, 8213] avg=8192.0000 max=8791 max_rank=4 min=6275 min_rank=1 imbalance_ratio=1.073120 max_load_reduction=17002 max_load_reduction_pct=0.659171 moved_to_replicas=40812 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[7099, 6275, 8789, 8789, 8791, 8790, 8790, 8213] avg=8192.0000 max=8791 max_rank=4 min=6275 min_rank=1 imbalance_ratio=1.073120 max_load_reduction=17002 max_load_reduction_pct=0.659171 moved_to_replicas=40812 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[1, 20240, 30163, 1, 1, 11337, 0, 3793] avg=8192.0000 max=30163 max_rank=2 min=0 min_rank=6 imbalance_ratio=3.682007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[6306, 28, 15036, 12763, 5610, 25793, 0, 0] avg=8192.0000 max=25793 max_rank=5 min=0 min_rank=6 imbalance_ratio=3.148560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[7099, 6275, 8789, 8789, 8791, 8790, 8790, 8213] avg=8192.0000 max=8791 max_rank=4 min=6275 min_rank=1 imbalance_ratio=1.073120 max_load_reduction=17002 max_load_reduction_pct=0.659171 moved_to_replicas=40812 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[1, 20240, 30163, 1, 1, 11337, 0, 3793] avg=8192.0000 max=30163 max_rank=2 min=0 min_rank=6 imbalance_ratio=3.682007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[1, 20240, 30163, 1, 1, 11337, 0, 3793] avg=8192.0000 max=30163 max_rank=2 min=0 min_rank=6 imbalance_ratio=3.682007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[1, 20240, 30163, 1, 1, 11337, 0, 3793] avg=8192.0000 max=30163 max_rank=2 min=0 min_rank=6 imbalance_ratio=3.682007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[1, 20240, 30163, 1, 1, 11337, 0, 3793] avg=8192.0000 max=30163 max_rank=2 min=0 min_rank=6 imbalance_ratio=3.682007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[7489, 11336, 11337, 8905, 1, 11337, 11338, 3793] avg=8192.0000 max=11338 max_rank=6 min=1 min_rank=4 imbalance_ratio=1.384033 max_load_reduction=18825 max_load_reduction_pct=0.624109 moved_to_replicas=49595 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[7099, 6275, 8789, 8789, 8791, 8790, 8790, 8213] avg=8192.0000 max=8791 max_rank=4 min=6275 min_rank=1 imbalance_ratio=1.073120 max_load_reduction=17002 max_load_reduction_pct=0.659171 moved_to_replicas=40812 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[1, 20240, 30163, 1, 1, 11337, 0, 3793] avg=8192.0000 max=30163 max_rank=2 min=0 min_rank=6 imbalance_ratio=3.682007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[7489, 11336, 11337, 8905, 1, 11337, 11338, 3793] avg=8192.0000 max=11338 max_rank=6 min=1 min_rank=4 imbalance_ratio=1.384033 max_load_reduction=18825 max_load_reduction_pct=0.624109 moved_to_replicas=49595 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[7489, 11336, 11337, 8905, 1, 11337, 11338, 3793] avg=8192.0000 max=11338 max_rank=6 min=1 min_rank=4 imbalance_ratio=1.384033 max_load_reduction=18825 max_load_reduction_pct=0.624109 moved_to_replicas=49595 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[7489, 11336, 11337, 8905, 1, 11337, 11338, 3793] avg=8192.0000 max=11338 max_rank=6 min=1 min_rank=4 imbalance_ratio=1.384033 max_load_reduction=18825 max_load_reduction_pct=0.624109 moved_to_replicas=49595 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[7489, 11336, 11337, 8905, 1, 11337, 11338, 3793] avg=8192.0000 max=11338 max_rank=6 min=1 min_rank=4 imbalance_ratio=1.384033 max_load_reduction=18825 max_load_reduction_pct=0.624109 moved_to_replicas=49595 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[1, 20240, 30163, 1, 1, 11337, 0, 3793] avg=8192.0000 max=30163 max_rank=2 min=0 min_rank=6 imbalance_ratio=3.682007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[7489, 11336, 11337, 8905, 1, 11337, 11338, 3793] avg=8192.0000 max=11338 max_rank=6 min=1 min_rank=4 imbalance_ratio=1.384033 max_load_reduction=18825 max_load_reduction_pct=0.624109 moved_to_replicas=49595 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[1, 0, 32767, 0, 31151, 1616, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[1, 20240, 30163, 1, 1, 11337, 0, 3793] avg=8192.0000 max=30163 max_rank=2 min=0 min_rank=6 imbalance_ratio=3.682007 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[1, 0, 32767, 0, 31151, 1616, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[1, 0, 32767, 0, 31151, 1616, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[7489, 11336, 11337, 8905, 1, 11337, 11338, 3793] avg=8192.0000 max=11338 max_rank=6 min=1 min_rank=4 imbalance_ratio=1.384033 max_load_reduction=18825 max_load_reduction_pct=0.624109 moved_to_replicas=49595 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[8274, 8276, 8275, 8272, 8274, 7947, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94404 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[1, 0, 32767, 0, 31151, 1616, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[1, 0, 32767, 0, 31151, 1616, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[1, 0, 32767, 0, 31151, 1616, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[7489, 11336, 11337, 8905, 1, 11337, 11338, 3793] avg=8192.0000 max=11338 max_rank=6 min=1 min_rank=4 imbalance_ratio=1.384033 max_load_reduction=18825 max_load_reduction_pct=0.624109 moved_to_replicas=49595 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[8274, 8276, 8275, 8272, 8274, 7947, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94404 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[8274, 8276, 8275, 8272, 8274, 7947, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94404 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[8274, 8276, 8275, 8272, 8274, 7947, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94404 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[8274, 8276, 8275, 8272, 8274, 7947, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94404 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[1, 0, 32767, 0, 31151, 1616, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[8274, 8276, 8275, 8272, 8274, 7947, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94404 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[1, 0, 0, 32766, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[1, 0, 32767, 0, 31151, 1616, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[1, 0, 0, 32766, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[1, 0, 0, 32766, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[8275, 8276, 8275, 8276, 8272, 7944, 8273, 7945] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[8274, 8276, 8275, 8272, 8274, 7947, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94404 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[1, 0, 0, 32766, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[1, 0, 0, 32766, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[1, 0, 0, 32766, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[8274, 8276, 8275, 8272, 8274, 7947, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94404 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[8275, 8276, 8275, 8276, 8272, 7944, 8273, 7945] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[8275, 8276, 8275, 8276, 8272, 7944, 8273, 7945] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[8275, 8276, 8275, 8276, 8272, 7944, 8273, 7945] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[8275, 8276, 8275, 8276, 8272, 7944, 8273, 7945] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[8275, 8276, 8275, 8276, 8272, 7944, 8273, 7945] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[1, 173, 32594, 32767, 0, 0, 0, 1] avg=8192.0000 max=32767 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[1, 0, 0, 32766, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[1, 0, 0, 32766, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[1, 173, 32594, 32767, 0, 0, 0, 1] avg=8192.0000 max=32767 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[1, 173, 32594, 32767, 0, 0, 0, 1] avg=8192.0000 max=32767 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[8275, 7946, 8274, 8276, 8275, 8272, 7944, 8274] avg=8192.0000 max=8276 max_rank=3 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97578 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[8275, 8276, 8275, 8276, 8272, 7944, 8273, 7945] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[1, 173, 32594, 32767, 0, 0, 0, 1] avg=8192.0000 max=32767 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[1, 173, 32594, 32767, 0, 0, 0, 1] avg=8192.0000 max=32767 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[1, 173, 32594, 32767, 0, 0, 0, 1] avg=8192.0000 max=32767 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[8275, 7946, 8274, 8276, 8275, 8272, 7944, 8274] avg=8192.0000 max=8276 max_rank=3 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97578 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[8275, 7946, 8274, 8276, 8275, 8272, 7944, 8274] avg=8192.0000 max=8276 max_rank=3 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97578 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[8275, 8276, 8275, 8276, 8272, 7944, 8273, 7945] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[8275, 7946, 8274, 8276, 8275, 8272, 7944, 8274] avg=8192.0000 max=8276 max_rank=3 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97578 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[8275, 7946, 8274, 8276, 8275, 8272, 7944, 8274] avg=8192.0000 max=8276 max_rank=3 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97578 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[8275, 7946, 8274, 8276, 8275, 8272, 7944, 8274] avg=8192.0000 max=8276 max_rank=3 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97578 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[0, 1, 1, 32761, 0, 32767, 0, 6] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[1, 173, 32594, 32767, 0, 0, 0, 1] avg=8192.0000 max=32767 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[0, 1, 1, 32761, 0, 32767, 0, 6] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[0, 1, 1, 32761, 0, 32767, 0, 6] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[1, 173, 32594, 32767, 0, 0, 0, 1] avg=8192.0000 max=32767 max_rank=3 min=0 min_rank=4 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[8275, 8275, 8275, 8273, 8276, 8272, 7944, 7946] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97875 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[0, 1, 1, 32761, 0, 32767, 0, 6] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[8275, 7946, 8274, 8276, 8275, 8272, 7944, 8274] avg=8192.0000 max=8276 max_rank=3 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97578 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[0, 1, 1, 32761, 0, 32767, 0, 6] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[8275, 8275, 8275, 8273, 8276, 8272, 7944, 7946] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97875 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[0, 1, 1, 32761, 0, 32767, 0, 6] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[8275, 8275, 8275, 8273, 8276, 8272, 7944, 7946] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97875 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[8275, 7946, 8274, 8276, 8275, 8272, 7944, 8274] avg=8192.0000 max=8276 max_rank=3 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97578 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[8275, 8275, 8275, 8273, 8276, 8272, 7944, 7946] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97875 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[8275, 8275, 8275, 8273, 8276, 8272, 7944, 7946] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97875 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[110, 32657, 0, 0, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[8275, 8275, 8275, 8273, 8276, 8272, 7944, 7946] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97875 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[0, 1, 1, 32761, 0, 32767, 0, 6] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[110, 32657, 0, 0, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[110, 32657, 0, 0, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[0, 1, 1, 32761, 0, 32767, 0, 6] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[110, 32657, 0, 0, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[7948, 8274, 8276, 8275, 8272, 7944, 8274, 8273] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97523 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[8275, 8275, 8275, 8273, 8276, 8272, 7944, 7946] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97875 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[7948, 8274, 8276, 8275, 8272, 7944, 8274, 8273] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97523 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[7948, 8274, 8276, 8275, 8272, 7944, 8274, 8273] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97523 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[110, 32657, 0, 0, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[110, 32657, 0, 0, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[8275, 8275, 8275, 8273, 8276, 8272, 7944, 7946] avg=8192.0000 max=8276 max_rank=4 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97875 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[7948, 8274, 8276, 8275, 8272, 7944, 8274, 8273] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97523 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[7948, 8274, 8276, 8275, 8272, 7944, 8274, 8273] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97523 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[32767, 0, 0, 1, 1, 8, 0, 32759] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[7948, 8274, 8276, 8275, 8272, 7944, 8274, 8273] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97523 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[32767, 0, 0, 1, 1, 8, 0, 32759] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[32767, 0, 0, 1, 1, 8, 0, 32759] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[110, 32657, 0, 0, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[32767, 0, 0, 1, 1, 8, 0, 32759] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[8275, 8276, 8272, 8274, 8274, 7947, 7944, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[110, 32657, 0, 0, 32767, 0, 1, 1] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[8275, 8276, 8272, 8274, 8274, 7947, 7944, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[8275, 8276, 8272, 8274, 8274, 7947, 7944, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[7948, 8274, 8276, 8275, 8272, 7944, 8274, 8273] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97523 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[32767, 0, 0, 1, 1, 8, 0, 32759] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[32767, 0, 0, 1, 1, 8, 0, 32759] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[8275, 8276, 8272, 8274, 8274, 7947, 7944, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[7948, 8274, 8276, 8275, 8272, 7944, 8274, 8273] avg=8192.0000 max=8276 max_rank=2 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97523 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[8275, 8276, 8272, 8274, 8274, 7947, 7944, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[0, 0, 0, 32767, 1, 1, 0, 32767] avg=8192.0000 max=32767 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[8275, 8276, 8272, 8274, 8274, 7947, 7944, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[0, 0, 0, 32767, 1, 1, 0, 32767] avg=8192.0000 max=32767 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[0, 0, 0, 32767, 1, 1, 0, 32767] avg=8192.0000 max=32767 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[32767, 0, 0, 1, 1, 8, 0, 32759] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[0, 0, 0, 32767, 1, 1, 0, 32767] avg=8192.0000 max=32767 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[8275, 8276, 7944, 8272, 8273, 7946, 8275, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[32767, 0, 0, 1, 1, 8, 0, 32759] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[8275, 8276, 7944, 8272, 8273, 7946, 8275, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[8275, 8276, 7944, 8272, 8273, 7946, 8275, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[0, 0, 0, 32767, 1, 1, 0, 32767] avg=8192.0000 max=32767 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[8275, 8276, 8272, 8274, 8274, 7947, 7944, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[0, 0, 0, 32767, 1, 1, 0, 32767] avg=8192.0000 max=32767 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[8275, 8276, 7944, 8272, 8273, 7946, 8275, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[8275, 8276, 8272, 8274, 8274, 7947, 7944, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=6 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97959 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[30591, 0, 1, 0, 2176, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[8275, 8276, 7944, 8272, 8273, 7946, 8275, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[8275, 8276, 7944, 8272, 8273, 7946, 8275, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[30591, 0, 1, 0, 2176, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[30591, 0, 1, 0, 2176, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[0, 0, 0, 32767, 1, 1, 0, 32767] avg=8192.0000 max=32767 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[30591, 0, 1, 0, 2176, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[8274, 8276, 8274, 8275, 7947, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=92871 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[8274, 8276, 8274, 8275, 7947, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=92871 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[0, 0, 0, 32767, 1, 1, 0, 32767] avg=8192.0000 max=32767 max_rank=3 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[30591, 0, 1, 0, 2176, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[8274, 8276, 8274, 8275, 7947, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=92871 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[8275, 8276, 7944, 8272, 8273, 7946, 8275, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[30591, 0, 1, 0, 2176, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[8274, 8276, 8274, 8275, 7947, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=92871 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[8275, 8276, 7944, 8272, 8273, 7946, 8275, 8275] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97965 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[1, 0, 347, 1, 32767, 32416, 4, 0] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[8274, 8276, 8274, 8275, 7947, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=92871 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[1, 0, 347, 1, 32767, 32416, 4, 0] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[8274, 8276, 8274, 8275, 7947, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=92871 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[1, 0, 347, 1, 32767, 32416, 4, 0] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[30591, 0, 1, 0, 2176, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[1, 0, 347, 1, 32767, 32416, 4, 0] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[7948, 8276, 7946, 8274, 8272, 8273, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7946 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97259 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[7948, 8276, 7946, 8274, 8272, 8273, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7946 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97259 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[30591, 0, 1, 0, 2176, 32767, 1, 0] avg=8192.0000 max=32767 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[7948, 8276, 7946, 8274, 8272, 8273, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7946 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97259 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[1, 0, 347, 1, 32767, 32416, 4, 0] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[8274, 8276, 8274, 8275, 7947, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=92871 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[1, 0, 347, 1, 32767, 32416, 4, 0] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[7948, 8276, 7946, 8274, 8272, 8273, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7946 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97259 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[1, 0, 1, 0, 0, 32765, 6, 32763] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[8274, 8276, 8274, 8275, 7947, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=92871 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[7948, 8276, 7946, 8274, 8272, 8273, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7946 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97259 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[1, 0, 1, 0, 0, 32765, 6, 32763] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[7948, 8276, 7946, 8274, 8272, 8273, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7946 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97259 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[1, 0, 1, 0, 0, 32765, 6, 32763] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[1, 0, 347, 1, 32767, 32416, 4, 0] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[8274, 8275, 8273, 8272, 7944, 8274, 7950, 8274] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[1, 0, 1, 0, 0, 32765, 6, 32763] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[8274, 8275, 8273, 8272, 7944, 8274, 7950, 8274] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[8274, 8275, 8273, 8272, 7944, 8274, 7950, 8274] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[1, 0, 347, 1, 32767, 32416, 4, 0] avg=8192.0000 max=32767 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[1, 0, 1, 0, 0, 32765, 6, 32763] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[7948, 8276, 7946, 8274, 8272, 8273, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7946 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97259 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[1, 0, 1, 0, 0, 32765, 6, 32763] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[8274, 8275, 8273, 8272, 7944, 8274, 7950, 8274] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[32751, 0, 32767, 1, 16, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[7948, 8276, 7946, 8274, 8272, 8273, 8275, 8272] avg=8192.0000 max=8276 max_rank=1 min=7946 min_rank=2 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97259 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[32751, 0, 32767, 1, 16, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[32751, 0, 32767, 1, 16, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[8274, 8275, 8273, 8272, 7944, 8274, 7950, 8274] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[8274, 8275, 8273, 8272, 7944, 8274, 7950, 8274] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[8273, 8276, 8275, 8274, 7948, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[1, 0, 1, 0, 0, 32765, 6, 32763] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[8273, 8276, 8275, 8274, 7948, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[32751, 0, 32767, 1, 16, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[8273, 8276, 8275, 8274, 7948, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[1, 0, 1, 0, 0, 32765, 6, 32763] avg=8192.0000 max=32765 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[32751, 0, 32767, 1, 16, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[8274, 8275, 8273, 8272, 7944, 8274, 7950, 8274] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[32751, 0, 32767, 1, 16, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[32767, 0, 1, 4, 924, 0, 31840, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[8273, 8276, 8275, 8274, 7948, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[32767, 0, 1, 4, 924, 0, 31840, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[32767, 0, 1, 4, 924, 0, 31840, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[8274, 8275, 8273, 8272, 7944, 8274, 7950, 8274] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=4 imbalance_ratio=1.010132 max_load_reduction=24490 max_load_reduction_pct=0.747444 moved_to_replicas=97950 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[8273, 8276, 8275, 8274, 7948, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[8273, 8276, 8275, 8274, 7948, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[8275, 8276, 8274, 8274, 7947, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=96350 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[32751, 0, 32767, 1, 16, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[8275, 8276, 8274, 8274, 7947, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=96350 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[8275, 8276, 8274, 8274, 7947, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=96350 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[32767, 0, 1, 4, 924, 0, 31840, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[32751, 0, 32767, 1, 16, 0, 1, 0] avg=8192.0000 max=32767 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[32767, 0, 1, 4, 924, 0, 31840, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[32767, 0, 1, 4, 924, 0, 31840, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[8273, 8276, 8275, 8274, 7948, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[8275, 8276, 8274, 8274, 7947, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=96350 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[0, 1, 31466, 0, 2, 1, 1299, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[0, 1, 31466, 0, 2, 1, 1299, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[0, 1, 31466, 0, 2, 1, 1299, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[8275, 8276, 8274, 8274, 7947, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=96350 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[8273, 8276, 8275, 8274, 7948, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[8275, 8276, 8274, 8274, 7947, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=96350 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[8275, 7945, 8274, 8276, 8274, 8274, 7946, 8272] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[8275, 7945, 8274, 8276, 8274, 8274, 7946, 8272] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[8275, 7945, 8274, 8276, 8274, 8274, 7946, 8272] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[32767, 0, 1, 4, 924, 0, 31840, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[0, 1, 31466, 0, 2, 1, 1299, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[0, 1, 31466, 0, 2, 1, 1299, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[8275, 8276, 8274, 8274, 7947, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=96350 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[32767, 0, 1, 4, 924, 0, 31840, 0] avg=8192.0000 max=32767 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[0, 1, 31466, 0, 2, 1, 1299, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[1, 32767, 0, 0, 32590, 0, 1, 177] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[8275, 7945, 8274, 8276, 8274, 8274, 7946, 8272] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[1, 32767, 0, 0, 32590, 0, 1, 177] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[1, 32767, 0, 0, 32590, 0, 1, 177] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[8275, 7945, 8274, 8276, 8274, 8274, 7946, 8272] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[8275, 8276, 8274, 8274, 7947, 8272, 8274, 7944] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=96350 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[8276, 8276, 8275, 8272, 8275, 7944, 8273, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97487 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[8275, 7945, 8274, 8276, 8274, 8274, 7946, 8272] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[8276, 8276, 8275, 8272, 8275, 7944, 8273, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97487 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[8276, 8276, 8275, 8272, 8275, 7944, 8273, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97487 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[0, 1, 31466, 0, 2, 1, 1299, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[1, 32767, 0, 0, 32590, 0, 1, 177] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[1, 32767, 0, 0, 32590, 0, 1, 177] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[8275, 7945, 8274, 8276, 8274, 8274, 7946, 8272] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[0, 1, 31466, 0, 2, 1, 1299, 32767] avg=8192.0000 max=32767 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[1, 32767, 0, 0, 32590, 0, 1, 177] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[8276, 8276, 8275, 8272, 8275, 7944, 8273, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97487 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[8276, 8276, 8275, 8272, 8275, 7944, 8273, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97487 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[8275, 7945, 8274, 8276, 8274, 8274, 7946, 8272] avg=8192.0000 max=8276 max_rank=3 min=7945 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=94940 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[8276, 8276, 8275, 8272, 8275, 7944, 8273, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97487 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[1, 32767, 0, 0, 32590, 0, 1, 177] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[8276, 8276, 8275, 8272, 8275, 7944, 8273, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97487 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[1, 32767, 0, 0, 32590, 0, 1, 177] avg=8192.0000 max=32767 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[8276, 8276, 8275, 8272, 8275, 7944, 8273, 7945] avg=8192.0000 max=8276 max_rank=0 min=7944 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24491 max_load_reduction_pct=0.747429 moved_to_replicas=97487 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=32 total_expected_recv=244219 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=227456 peak_demand=8275 required_capacity=1.010132 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=32 total_expected_recv=274817 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=196992 peak_demand=13590 required_capacity=1.658936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=32 total_expected_recv=276976 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=195328 peak_demand=13430 required_capacity=1.639404 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=32 total_expected_recv=266836 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=205184 peak_demand=12574 required_capacity=1.534912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=32 total_expected_recv=244341 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=227584 peak_demand=8791 required_capacity=1.073120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=32 total_expected_recv=260740 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=211456 peak_demand=13055 required_capacity=1.593628 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=32 total_expected_recv=275816 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=195072 peak_demand=13591 required_capacity=1.659058 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=32 total_expected_recv=253407 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=217728 peak_demand=8276 required_capacity=1.010254 + [2026-08-30 05:09:20] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 914.6 | throughput per GPU (TFLOP/s/GPU): 364.5 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.379854E+00 | loss scale: 1.0 | grad norm: 4.787 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 14208 | min safe capacity: 1.7344 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=14848 expected_recv=25625 expected_overflow=0 exact_recv_per_rank=[6535, 2222, 12561, 12561, 2472, 12563, 12560, 4062] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=14848 expected_recv=26943 expected_overflow=0 exact_recv_per_rank=[6535, 2222, 12561, 12561, 2472, 12563, 12560, 4062] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=14848 expected_recv=26448 expected_overflow=0 exact_recv_per_rank=[6535, 2222, 12561, 12561, 2472, 12563, 12560, 4062] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=14848 expected_recv=2251 expected_overflow=0 exact_recv_per_rank=[6535, 2222, 12561, 12561, 2472, 12563, 12560, 4062] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=14848 expected_recv=12747 expected_overflow=0 exact_recv_per_rank=[6535, 2222, 12561, 12561, 2472, 12563, 12560, 4062] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=14848 expected_recv=12564 expected_overflow=0 exact_recv_per_rank=[6535, 2222, 12561, 12561, 2472, 12563, 12560, 4062] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=14848 expected_recv=18442 expected_overflow=0 exact_recv_per_rank=[6535, 2222, 12561, 12561, 2472, 12563, 12560, 4062] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=14848 expected_recv=25625 actual_valid_recv=25856 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=14848 expected_recv=6052 expected_overflow=0 exact_recv_per_rank=[6535, 2222, 12561, 12561, 2472, 12563, 12560, 4062] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=14848 expected_recv=26943 actual_valid_recv=27136 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=14848 expected_recv=26448 actual_valid_recv=26624 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=14848 expected_recv=2251 actual_valid_recv=2560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=14848 expected_recv=12747 actual_valid_recv=12928 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=25856 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=14848 expected_recv=12564 actual_valid_recv=12928 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=14848 expected_recv=18442 actual_valid_recv=18816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=27136 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=26624 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=2560 slack_slots=27136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=12928 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=14848 expected_recv=6052 actual_valid_recv=6144 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=12928 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=18816 slack_slots=10880 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=2 received_num_tokens=29696 valid_num_tokens=6144 slack_slots=23552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=14848 expected_recv=15866 expected_overflow=0 exact_recv_per_rank=[8275, 7969, 8274, 7923, 8274, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 7969, 8274, 7923, 8274, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 7969, 8274, 7923, 8274, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=14848 expected_recv=15917 expected_overflow=0 exact_recv_per_rank=[8275, 7969, 8274, 7923, 8274, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8275, 7969, 8274, 7923, 8274, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=14848 expected_recv=15866 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 7969, 8274, 7923, 8274, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 7969, 8274, 7923, 8274, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 7969, 8274, 7923, 8274, 8274, 8273, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=14848 expected_recv=15917 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=14848 expected_recv=15879 expected_overflow=0 exact_recv_per_rank=[8226, 8603, 8601, 6949, 8601, 8602, 7351, 8603] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=14848 expected_recv=17532 expected_overflow=0 exact_recv_per_rank=[8226, 8603, 8601, 6949, 8601, 8602, 7351, 8603] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=14848 expected_recv=17530 expected_overflow=0 exact_recv_per_rank=[8226, 8603, 8601, 6949, 8601, 8602, 7351, 8603] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=14848 expected_recv=17508 expected_overflow=0 exact_recv_per_rank=[8226, 8603, 8601, 6949, 8601, 8602, 7351, 8603] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=14848 expected_recv=14245 expected_overflow=0 exact_recv_per_rank=[8226, 8603, 8601, 6949, 8601, 8602, 7351, 8603] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=14848 expected_recv=14566 expected_overflow=0 exact_recv_per_rank=[8226, 8603, 8601, 6949, 8601, 8602, 7351, 8603] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=14848 expected_recv=17531 expected_overflow=0 exact_recv_per_rank=[8226, 8603, 8601, 6949, 8601, 8602, 7351, 8603] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=14848 expected_recv=15879 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=14848 expected_recv=17532 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=14848 expected_recv=17530 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=14848 expected_recv=17508 actual_valid_recv=17792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=14848 expected_recv=14245 actual_valid_recv=14592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=14848 expected_recv=14566 actual_valid_recv=14848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=14848 expected_recv=16281 expected_overflow=0 exact_recv_per_rank=[8226, 8603, 8601, 6949, 8601, 8602, 7351, 8603] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=14848 expected_recv=17531 actual_valid_recv=17664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=17664 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=14848 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=17664 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=17792 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=14592 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=14848 expected_recv=16281 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=17664 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=14848 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[7943, 8275, 8276, 8272, 8275, 8272, 7949, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[7943, 8275, 8276, 8272, 8275, 8272, 7949, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7943, 8275, 8276, 8272, 8275, 8272, 7949, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=14848 expected_recv=16545 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[7943, 8275, 8276, 8272, 8275, 8272, 7949, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=14848 expected_recv=16212 expected_overflow=0 exact_recv_per_rank=[7943, 8275, 8276, 8272, 8275, 8272, 7949, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=14848 expected_recv=15899 expected_overflow=0 exact_recv_per_rank=[7943, 8275, 8276, 8272, 8275, 8272, 7949, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7943, 8275, 8276, 8272, 8275, 8272, 7949, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=14848 expected_recv=16222 expected_overflow=0 exact_recv_per_rank=[7943, 8275, 8276, 8272, 8275, 8272, 7949, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=14848 expected_recv=15899 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=14848 expected_recv=16212 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=14848 expected_recv=16222 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7951, 8275, 8274, 8274, 8275, 8272, 7943, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7951, 8275, 8274, 8274, 8275, 8272, 7943, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[7951, 8275, 8274, 8274, 8275, 8272, 7943, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[7951, 8275, 8274, 8274, 8275, 8272, 7943, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=14848 expected_recv=15889 expected_overflow=0 exact_recv_per_rank=[7951, 8275, 8274, 8274, 8275, 8272, 7943, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7951, 8275, 8274, 8274, 8275, 8272, 7943, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7951, 8275, 8274, 8274, 8275, 8272, 7943, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=14848 expected_recv=15898 expected_overflow=0 exact_recv_per_rank=[7951, 8275, 8274, 8274, 8275, 8272, 7943, 8272] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=14848 expected_recv=15889 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=14848 expected_recv=16222 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7948, 8274, 8272, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=14848 expected_recv=15898 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=14848 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7948, 8274, 8272, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=14848 expected_recv=16222 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=14848 expected_recv=16545 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7948, 8274, 8272, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=14848 expected_recv=16231 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7948, 8274, 8272, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7948, 8274, 8272, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=14848 expected_recv=16231 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=14848 expected_recv=16211 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7948, 8274, 8272, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7948, 8274, 8272, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=14848 expected_recv=16218 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8275, 7948, 8274, 8272, 7944, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7949, 8275, 8273, 8274, 8273, 7944, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=14848 expected_recv=16211 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=14848 expected_recv=16218 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=14848 expected_recv=15888 expected_overflow=0 exact_recv_per_rank=[7949, 8275, 8273, 8274, 8273, 7944, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=14848 expected_recv=15888 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7949, 8275, 8273, 8274, 8273, 7944, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7949, 8275, 8273, 8274, 8273, 7944, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=14848 expected_recv=15897 expected_overflow=0 exact_recv_per_rank=[7949, 8275, 8273, 8274, 8273, 7944, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7949, 8275, 8273, 8274, 8273, 7944, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=14848 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[7949, 8275, 8273, 8274, 8273, 7944, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=14848 expected_recv=15897 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7949, 8275, 8273, 8274, 8273, 7944, 8274, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 8274, 7948, 7940, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=14848 expected_recv=16545 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=14848 expected_recv=16213 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 8274, 7948, 7940, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 8274, 7948, 7940, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=14848 expected_recv=16213 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 8274, 7948, 7940, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 8274, 7948, 7940, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 8274, 7948, 7940, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=14848 expected_recv=15906 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 8274, 7948, 7940, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=14848 expected_recv=16209 expected_overflow=0 exact_recv_per_rank=[8275, 8275, 8274, 8274, 7948, 7940, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=14848 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[7944, 8276, 8272, 8273, 8274, 7948, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=14848 expected_recv=15906 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=14848 expected_recv=16209 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=14848 expected_recv=15905 expected_overflow=0 exact_recv_per_rank=[7944, 8276, 8272, 8273, 8274, 7948, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=14848 expected_recv=16545 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[7944, 8276, 8272, 8273, 8274, 7948, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7944, 8276, 8272, 8273, 8274, 7948, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=14848 expected_recv=15905 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=14848 expected_recv=16219 expected_overflow=0 exact_recv_per_rank=[7944, 8276, 8272, 8273, 8274, 7948, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[7944, 8276, 8272, 8273, 8274, 7948, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[7944, 8276, 8272, 8273, 8274, 7948, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=14848 expected_recv=16219 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=14848 expected_recv=16211 expected_overflow=0 exact_recv_per_rank=[7944, 8276, 8272, 8273, 8274, 7948, 8275, 8274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=14848 expected_recv=16209 expected_overflow=0 exact_recv_per_rank=[7950, 8275, 8273, 8274, 8274, 8272, 8275, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=14848 expected_recv=16545 expected_overflow=0 exact_recv_per_rank=[7950, 8275, 8273, 8274, 8274, 8272, 8275, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=14848 expected_recv=16209 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=14848 expected_recv=16211 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=14848 expected_recv=16545 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=14848 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[7950, 8275, 8273, 8274, 8274, 8272, 8275, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[7950, 8275, 8273, 8274, 8274, 8272, 8275, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=14848 expected_recv=16217 actual_valid_recv=16384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16384 slack_slots=13312 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=14848 expected_recv=15906 expected_overflow=0 exact_recv_per_rank=[7950, 8275, 8273, 8274, 8274, 8272, 8275, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7950, 8275, 8273, 8274, 8274, 8272, 8275, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[7950, 8275, 8273, 8274, 8274, 8272, 8275, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=14848 expected_recv=15906 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[7950, 8275, 8273, 8274, 8274, 8272, 8275, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=14848 expected_recv=15882 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8275, 7943, 8275, 7950, 8273, 8271] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=14848 expected_recv=15902 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8275, 7943, 8275, 7950, 8273, 8271] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=14848 expected_recv=15882 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8275, 7943, 8275, 7950, 8273, 8271] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=14848 expected_recv=15902 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8275, 7943, 8275, 7950, 8273, 8271] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8275, 7943, 8275, 7950, 8273, 8271] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8275, 7943, 8275, 7950, 8273, 8271] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8275, 7943, 8275, 7950, 8273, 8271] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8275, 7943, 8275, 7950, 8273, 8271] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=14848 expected_recv=21922 expected_overflow=0 exact_recv_per_rank=[12594, 12607, 4, 12607, 2502, 12607, 8, 12607] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=14848 expected_recv=21923 expected_overflow=0 exact_recv_per_rank=[12594, 12607, 4, 12607, 2502, 12607, 8, 12607] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=14848 expected_recv=21924 expected_overflow=0 exact_recv_per_rank=[12594, 12607, 4, 12607, 2502, 12607, 8, 12607] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=14848 expected_recv=4814 expected_overflow=0 exact_recv_per_rank=[12594, 12607, 4, 12607, 2502, 12607, 8, 12607] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=14848 expected_recv=21922 actual_valid_recv=22016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=14848 expected_recv=21923 actual_valid_recv=22016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=14848 expected_recv=21924 actual_valid_recv=22016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=22016 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=14848 expected_recv=4814 actual_valid_recv=5120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=22016 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=22016 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=14848 expected_recv=21910 expected_overflow=0 exact_recv_per_rank=[12594, 12607, 4, 12607, 2502, 12607, 8, 12607] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=5120 slack_slots=24576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=14848 expected_recv=11815 expected_overflow=0 exact_recv_per_rank=[12594, 12607, 4, 12607, 2502, 12607, 8, 12607] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=14848 expected_recv=21910 actual_valid_recv=22272 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=14848 expected_recv=21923 expected_overflow=0 exact_recv_per_rank=[12594, 12607, 4, 12607, 2502, 12607, 8, 12607] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=14848 expected_recv=11815 actual_valid_recv=12032 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=22272 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=12032 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=14848 expected_recv=4841 expected_overflow=0 exact_recv_per_rank=[12594, 12607, 4, 12607, 2502, 12607, 8, 12607] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=14848 expected_recv=21923 actual_valid_recv=22016 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7951, 8272, 8273, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7951, 8272, 8273, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=14848 expected_recv=16217 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7951, 8272, 8273, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=22016 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=14848 expected_recv=4841 actual_valid_recv=5120 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7951, 8272, 8273, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=14848 expected_recv=16217 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=2 received_num_tokens=29696 valid_num_tokens=5120 slack_slots=24576 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7951, 8272, 8273, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=14848 expected_recv=15905 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7951, 8272, 8273, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=14848 expected_recv=15905 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7951, 8272, 8273, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=14848 expected_recv=16212 expected_overflow=0 exact_recv_per_rank=[8274, 8275, 8274, 8274, 7951, 8272, 8273, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=14848 expected_recv=15908 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 7950, 8274, 8272, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=14848 expected_recv=16544 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 7950, 8274, 8272, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=14848 expected_recv=16212 actual_valid_recv=16512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=14848 expected_recv=15908 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=14848 expected_recv=15878 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 7950, 8274, 8272, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=14848 expected_recv=16544 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 7950, 8274, 8272, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=2 received_num_tokens=29696 valid_num_tokens=16512 slack_slots=13184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=14848 expected_recv=15878 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 7950, 8274, 8272, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 7950, 8274, 8272, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 7950, 8274, 8272, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 7950, 8274, 8272, 8274, 7943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=14848 expected_recv=15878 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 7943, 7946, 8274, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 7943, 7946, 8274, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 7943, 7946, 8274, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16768 slack_slots=12928 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 7943, 7946, 8274, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=14848 expected_recv=15878 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 7943, 7946, 8274, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=14848 expected_recv=15903 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 7943, 7946, 8274, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=14848 expected_recv=15903 actual_valid_recv=16128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16128 slack_slots=13568 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 7943, 7946, 8274, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=14848 expected_recv=16549 expected_overflow=0 exact_recv_per_rank=[8275, 8274, 8274, 7943, 7946, 8274, 8275, 8275] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=14848 expected_recv=16547 expected_overflow=0 exact_recv_per_rank=[8274, 7950, 8276, 8276, 8272, 7941, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=14848 expected_recv=15875 expected_overflow=0 exact_recv_per_rank=[8274, 7950, 8276, 8276, 8272, 7941, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=14848 expected_recv=16550 expected_overflow=0 exact_recv_per_rank=[8274, 7950, 8276, 8276, 8272, 7941, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=14848 expected_recv=16551 expected_overflow=0 exact_recv_per_rank=[8274, 7950, 8276, 8276, 8272, 7941, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=14848 expected_recv=16549 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=14848 expected_recv=16547 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=14848 expected_recv=15875 actual_valid_recv=16256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=14848 expected_recv=16550 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=14848 expected_recv=16231 expected_overflow=0 exact_recv_per_rank=[8274, 7950, 8276, 8276, 8272, 7941, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=14848 expected_recv=16546 expected_overflow=0 exact_recv_per_rank=[8274, 7950, 8276, 8276, 8272, 7941, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=14848 expected_recv=16551 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16256 slack_slots=13440 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=14848 expected_recv=16231 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=14848 expected_recv=16546 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=14848 expected_recv=16224 expected_overflow=0 exact_recv_per_rank=[8274, 7950, 8276, 8276, 8272, 7941, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=14848 expected_recv=16224 actual_valid_recv=16640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=14848 expected_recv=16548 expected_overflow=0 exact_recv_per_rank=[8274, 7950, 8276, 8276, 8272, 7941, 8274, 8273] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16640 slack_slots=13056 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[6535, 42, 12561, 14741, 2472, 29175, 3, 7] avg=8192.0000 max=29175 max_rank=5 min=3 min_rank=6 imbalance_ratio=3.561401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[6535, 42, 12561, 14741, 2472, 29175, 3, 7] avg=8192.0000 max=29175 max_rank=5 min=3 min_rank=6 imbalance_ratio=3.561401 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=14848 expected_recv=16548 actual_valid_recv=16896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[6535, 42, 12561, 14741, 2472, 29175, 3, 7] avg=8192.0000 max=29175 max_rank=5 min=3 min_rank=6 imbalance_ratio=3.561401 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=15 calls=2 received_num_tokens=29696 valid_num_tokens=16896 slack_slots=12800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[6535, 42, 12561, 14741, 2472, 29175, 3, 7] avg=8192.0000 max=29175 max_rank=5 min=3 min_rank=6 imbalance_ratio=3.561401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[6535, 42, 12561, 14741, 2472, 29175, 3, 7] avg=8192.0000 max=29175 max_rank=5 min=3 min_rank=6 imbalance_ratio=3.561401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[6535, 42, 12561, 14741, 2472, 29175, 3, 7] avg=8192.0000 max=29175 max_rank=5 min=3 min_rank=6 imbalance_ratio=3.561401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[6535, 2222, 12561, 12561, 2472, 12563, 12560, 4062] avg=8192.0000 max=12563 max_rank=5 min=2222 min_rank=1 imbalance_ratio=1.533569 max_load_reduction=16612 max_load_reduction_pct=0.569392 moved_to_replicas=33172 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[6535, 2222, 12561, 12561, 2472, 12563, 12560, 4062] avg=8192.0000 max=12563 max_rank=5 min=2222 min_rank=1 imbalance_ratio=1.533569 max_load_reduction=16612 max_load_reduction_pct=0.569392 moved_to_replicas=33172 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[6535, 2222, 12561, 12561, 2472, 12563, 12560, 4062] avg=8192.0000 max=12563 max_rank=5 min=2222 min_rank=1 imbalance_ratio=1.533569 max_load_reduction=16612 max_load_reduction_pct=0.569392 moved_to_replicas=33172 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[6535, 42, 12561, 14741, 2472, 29175, 3, 7] avg=8192.0000 max=29175 max_rank=5 min=3 min_rank=6 imbalance_ratio=3.561401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[6535, 2222, 12561, 12561, 2472, 12563, 12560, 4062] avg=8192.0000 max=12563 max_rank=5 min=2222 min_rank=1 imbalance_ratio=1.533569 max_load_reduction=16612 max_load_reduction_pct=0.569392 moved_to_replicas=33172 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[6535, 2222, 12561, 12561, 2472, 12563, 12560, 4062] avg=8192.0000 max=12563 max_rank=5 min=2222 min_rank=1 imbalance_ratio=1.533569 max_load_reduction=16612 max_load_reduction_pct=0.569392 moved_to_replicas=33172 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[6535, 2222, 12561, 12561, 2472, 12563, 12560, 4062] avg=8192.0000 max=12563 max_rank=5 min=2222 min_rank=1 imbalance_ratio=1.533569 max_load_reduction=16612 max_load_reduction_pct=0.569392 moved_to_replicas=33172 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[6535, 42, 12561, 14741, 2472, 29175, 3, 7] avg=8192.0000 max=29175 max_rank=5 min=3 min_rank=6 imbalance_ratio=3.561401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[3, 6744, 22, 4, 5, 26020, 0, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=6 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[3, 6744, 22, 4, 5, 26020, 0, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=6 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[6535, 2222, 12561, 12561, 2472, 12563, 12560, 4062] avg=8192.0000 max=12563 max_rank=5 min=2222 min_rank=1 imbalance_ratio=1.533569 max_load_reduction=16612 max_load_reduction_pct=0.569392 moved_to_replicas=33172 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[3, 6744, 22, 4, 5, 26020, 0, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=6 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[3, 6744, 22, 4, 5, 26020, 0, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=6 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[3, 6744, 22, 4, 5, 26020, 0, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=6 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[6535, 2222, 12561, 12561, 2472, 12563, 12560, 4062] avg=8192.0000 max=12563 max_rank=5 min=2222 min_rank=1 imbalance_ratio=1.533569 max_load_reduction=16612 max_load_reduction_pct=0.569392 moved_to_replicas=33172 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[3, 6744, 22, 4, 5, 26020, 0, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=6 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[8275, 7969, 8274, 7923, 8274, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=0 min=7923 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=84773 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[8275, 7969, 8274, 7923, 8274, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=0 min=7923 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=84773 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[8275, 7969, 8274, 7923, 8274, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=0 min=7923 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=84773 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[8275, 7969, 8274, 7923, 8274, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=0 min=7923 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=84773 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[3, 6744, 22, 4, 5, 26020, 0, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=6 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[8275, 7969, 8274, 7923, 8274, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=0 min=7923 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=84773 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[8275, 7969, 8274, 7923, 8274, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=0 min=7923 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=84773 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[3, 6744, 22, 4, 5, 26020, 0, 32738] avg=8192.0000 max=32738 max_rank=7 min=0 min_rank=6 imbalance_ratio=3.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[4, 0, 32755, 1, 16823, 15949, 4, 0] avg=8192.0000 max=32755 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[4, 0, 32755, 1, 16823, 15949, 4, 0] avg=8192.0000 max=32755 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[8275, 7969, 8274, 7923, 8274, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=0 min=7923 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=84773 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[4, 0, 32755, 1, 16823, 15949, 4, 0] avg=8192.0000 max=32755 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[4, 0, 32755, 1, 16823, 15949, 4, 0] avg=8192.0000 max=32755 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[4, 0, 32755, 1, 16823, 15949, 4, 0] avg=8192.0000 max=32755 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[8275, 7969, 8274, 7923, 8274, 8274, 8273, 8274] avg=8192.0000 max=8275 max_rank=0 min=7923 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24463 max_load_reduction_pct=0.747236 moved_to_replicas=84773 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[4, 0, 32755, 1, 16823, 15949, 4, 0] avg=8192.0000 max=32755 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[8226, 8603, 8601, 6949, 8601, 8602, 7351, 8603] avg=8192.0000 max=8603 max_rank=1 min=6949 min_rank=3 imbalance_ratio=1.050171 max_load_reduction=24152 max_load_reduction_pct=0.737353 moved_to_replicas=78448 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[8226, 8603, 8601, 6949, 8601, 8602, 7351, 8603] avg=8192.0000 max=8603 max_rank=1 min=6949 min_rank=3 imbalance_ratio=1.050171 max_load_reduction=24152 max_load_reduction_pct=0.737353 moved_to_replicas=78448 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[8226, 8603, 8601, 6949, 8601, 8602, 7351, 8603] avg=8192.0000 max=8603 max_rank=1 min=6949 min_rank=3 imbalance_ratio=1.050171 max_load_reduction=24152 max_load_reduction_pct=0.737353 moved_to_replicas=78448 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[8226, 8603, 8601, 6949, 8601, 8602, 7351, 8603] avg=8192.0000 max=8603 max_rank=1 min=6949 min_rank=3 imbalance_ratio=1.050171 max_load_reduction=24152 max_load_reduction_pct=0.737353 moved_to_replicas=78448 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[8226, 8603, 8601, 6949, 8601, 8602, 7351, 8603] avg=8192.0000 max=8603 max_rank=1 min=6949 min_rank=3 imbalance_ratio=1.050171 max_load_reduction=24152 max_load_reduction_pct=0.737353 moved_to_replicas=78448 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[4, 0, 32755, 1, 16823, 15949, 4, 0] avg=8192.0000 max=32755 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[8226, 8603, 8601, 6949, 8601, 8602, 7351, 8603] avg=8192.0000 max=8603 max_rank=1 min=6949 min_rank=3 imbalance_ratio=1.050171 max_load_reduction=24152 max_load_reduction_pct=0.737353 moved_to_replicas=78448 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[4, 0, 0, 32762, 32754, 6, 6, 4] avg=8192.0000 max=32762 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[4, 0, 32755, 1, 16823, 15949, 4, 0] avg=8192.0000 max=32755 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.998413 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[4, 0, 0, 32762, 32754, 6, 6, 4] avg=8192.0000 max=32762 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[8226, 8603, 8601, 6949, 8601, 8602, 7351, 8603] avg=8192.0000 max=8603 max_rank=1 min=6949 min_rank=3 imbalance_ratio=1.050171 max_load_reduction=24152 max_load_reduction_pct=0.737353 moved_to_replicas=78448 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[4, 0, 0, 32762, 32754, 6, 6, 4] avg=8192.0000 max=32762 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[4, 0, 0, 32762, 32754, 6, 6, 4] avg=8192.0000 max=32762 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[4, 0, 0, 32762, 32754, 6, 6, 4] avg=8192.0000 max=32762 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[4, 0, 0, 32762, 32754, 6, 6, 4] avg=8192.0000 max=32762 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[7943, 8275, 8276, 8272, 8275, 8272, 7949, 8274] avg=8192.0000 max=8276 max_rank=2 min=7943 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[8226, 8603, 8601, 6949, 8601, 8602, 7351, 8603] avg=8192.0000 max=8603 max_rank=1 min=6949 min_rank=3 imbalance_ratio=1.050171 max_load_reduction=24152 max_load_reduction_pct=0.737353 moved_to_replicas=78448 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[7943, 8275, 8276, 8272, 8275, 8272, 7949, 8274] avg=8192.0000 max=8276 max_rank=2 min=7943 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[7943, 8275, 8276, 8272, 8275, 8272, 7949, 8274] avg=8192.0000 max=8276 max_rank=2 min=7943 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[7943, 8275, 8276, 8272, 8275, 8272, 7949, 8274] avg=8192.0000 max=8276 max_rank=2 min=7943 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[7943, 8275, 8276, 8272, 8275, 8272, 7949, 8274] avg=8192.0000 max=8276 max_rank=2 min=7943 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[4, 0, 0, 32762, 32754, 6, 6, 4] avg=8192.0000 max=32762 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[7943, 8275, 8276, 8272, 8275, 8272, 7949, 8274] avg=8192.0000 max=8276 max_rank=2 min=7943 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[4, 32764, 0, 32764, 3, 0, 0, 1] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[4, 32764, 0, 32764, 3, 0, 0, 1] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[4, 0, 0, 32762, 32754, 6, 6, 4] avg=8192.0000 max=32762 max_rank=3 min=0 min_rank=1 imbalance_ratio=3.999268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[4, 32764, 0, 32764, 3, 0, 0, 1] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[4, 32764, 0, 32764, 3, 0, 0, 1] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[4, 32764, 0, 32764, 3, 0, 0, 1] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[7943, 8275, 8276, 8272, 8275, 8272, 7949, 8274] avg=8192.0000 max=8276 max_rank=2 min=7943 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[7951, 8275, 8274, 8274, 8275, 8272, 7943, 8272] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[7951, 8275, 8274, 8274, 8275, 8272, 7943, 8272] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[7943, 8275, 8276, 8272, 8275, 8272, 7949, 8274] avg=8192.0000 max=8276 max_rank=2 min=7943 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24486 max_load_reduction_pct=0.747390 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[4, 32764, 0, 32764, 3, 0, 0, 1] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[7951, 8275, 8274, 8274, 8275, 8272, 7943, 8272] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[7951, 8275, 8274, 8274, 8275, 8272, 7943, 8272] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[7951, 8275, 8274, 8274, 8275, 8272, 7943, 8272] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[7951, 8275, 8274, 8274, 8275, 8272, 7943, 8272] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[0, 3, 4, 6, 0, 32764, 1, 32758] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[0, 3, 4, 6, 0, 32764, 1, 32758] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[4, 32764, 0, 32764, 3, 0, 0, 1] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[4, 32764, 0, 32764, 3, 0, 0, 1] avg=8192.0000 max=32764 max_rank=1 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[0, 3, 4, 6, 0, 32764, 1, 32758] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[0, 3, 4, 6, 0, 32764, 1, 32758] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[0, 3, 4, 6, 0, 32764, 1, 32758] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[8275, 8274, 8275, 7948, 8274, 8272, 7944, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[8275, 8274, 8275, 7948, 8274, 8272, 7944, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[7951, 8275, 8274, 8274, 8275, 8272, 7943, 8272] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[7951, 8275, 8274, 8274, 8275, 8272, 7943, 8272] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[0, 3, 4, 6, 0, 32764, 1, 32758] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[8275, 8274, 8275, 7948, 8274, 8272, 7944, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[8275, 8274, 8275, 7948, 8274, 8272, 7944, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[8275, 8274, 8275, 7948, 8274, 8272, 7944, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[43, 0, 0, 32720, 32764, 1, 4, 4] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[8275, 8274, 8275, 7948, 8274, 8272, 7944, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[43, 0, 0, 32720, 32764, 1, 4, 4] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[0, 3, 4, 6, 0, 32764, 1, 32758] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[0, 3, 4, 6, 0, 32764, 1, 32758] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[43, 0, 0, 32720, 32764, 1, 4, 4] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[43, 0, 0, 32720, 32764, 1, 4, 4] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[7949, 8275, 8273, 8274, 8273, 7944, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97818 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[43, 0, 0, 32720, 32764, 1, 4, 4] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[7949, 8275, 8273, 8274, 8273, 7944, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97818 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[8275, 8274, 8275, 7948, 8274, 8272, 7944, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[43, 0, 0, 32720, 32764, 1, 4, 4] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[8275, 8274, 8275, 7948, 8274, 8272, 7944, 8274] avg=8192.0000 max=8275 max_rank=0 min=7944 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97933 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[7949, 8275, 8273, 8274, 8273, 7944, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97818 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[7949, 8275, 8273, 8274, 8273, 7944, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97818 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[7949, 8275, 8273, 8274, 8273, 7944, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97818 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[7949, 8275, 8273, 8274, 8273, 7944, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97818 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[32764, 0, 0, 4, 4, 0, 0, 32764] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[32764, 0, 0, 4, 4, 0, 0, 32764] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[43, 0, 0, 32720, 32764, 1, 4, 4] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[43, 0, 0, 32720, 32764, 1, 4, 4] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[32764, 0, 0, 4, 4, 0, 0, 32764] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[32764, 0, 0, 4, 4, 0, 0, 32764] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[8275, 8275, 8274, 8274, 7948, 7940, 8275, 8275] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[32764, 0, 0, 4, 4, 0, 0, 32764] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[8275, 8275, 8274, 8274, 7948, 7940, 8275, 8275] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[32764, 0, 0, 4, 4, 0, 0, 32764] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[7949, 8275, 8273, 8274, 8273, 7944, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97818 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[7949, 8275, 8273, 8274, 8273, 7944, 8274, 8274] avg=8192.0000 max=8275 max_rank=1 min=7944 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97818 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[8275, 8275, 8274, 8274, 7948, 7940, 8275, 8275] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[8275, 8275, 8274, 8274, 7948, 7940, 8275, 8275] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[8275, 8275, 8274, 8274, 7948, 7940, 8275, 8275] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[1, 1, 0, 32763, 3, 4, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[8275, 8275, 8274, 8274, 7948, 7940, 8275, 8275] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[1, 1, 0, 32763, 3, 4, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[32764, 0, 0, 4, 4, 0, 0, 32764] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[32764, 0, 0, 4, 4, 0, 0, 32764] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[1, 1, 0, 32763, 3, 4, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[1, 1, 0, 32763, 3, 4, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[7944, 8276, 8272, 8273, 8274, 7948, 8275, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[1, 1, 0, 32763, 3, 4, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[7944, 8276, 8272, 8273, 8274, 7948, 8275, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[8275, 8275, 8274, 8274, 7948, 7940, 8275, 8275] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[1, 1, 0, 32763, 3, 4, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[8275, 8275, 8274, 8274, 7948, 7940, 8275, 8275] avg=8192.0000 max=8275 max_rank=0 min=7940 min_rank=5 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97946 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[7944, 8276, 8272, 8273, 8274, 7948, 8275, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[7944, 8276, 8272, 8273, 8274, 7948, 8275, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[7944, 8276, 8272, 8273, 8274, 7948, 8275, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[7784, 0, 4, 0, 24980, 32764, 4, 0] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[7944, 8276, 8272, 8273, 8274, 7948, 8275, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[7784, 0, 4, 0, 24980, 32764, 4, 0] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[1, 1, 0, 32763, 3, 4, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[7784, 0, 4, 0, 24980, 32764, 4, 0] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[1, 1, 0, 32763, 3, 4, 0, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[7950, 8275, 8273, 8274, 8274, 8272, 8275, 7943] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=82974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[7950, 8275, 8273, 8274, 8274, 8272, 8275, 7943] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=82974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[7784, 0, 4, 0, 24980, 32764, 4, 0] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[7784, 0, 4, 0, 24980, 32764, 4, 0] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[7944, 8276, 8272, 8273, 8274, 7948, 8275, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[7784, 0, 4, 0, 24980, 32764, 4, 0] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[7950, 8275, 8273, 8274, 8274, 8272, 8275, 7943] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=82974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[7944, 8276, 8272, 8273, 8274, 7948, 8275, 8274] avg=8192.0000 max=8276 max_rank=1 min=7944 min_rank=0 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97945 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[7950, 8275, 8273, 8274, 8274, 8272, 8275, 7943] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=82974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[7950, 8275, 8273, 8274, 8274, 8272, 8275, 7943] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=82974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[4, 0, 23285, 3, 32761, 6829, 2654, 0] avg=8192.0000 max=32761 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[4, 0, 23285, 3, 32761, 6829, 2654, 0] avg=8192.0000 max=32761 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[7950, 8275, 8273, 8274, 8274, 8272, 8275, 7943] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=82974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[7784, 0, 4, 0, 24980, 32764, 4, 0] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[8274, 8275, 8275, 7943, 8275, 7950, 8273, 8271] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=79188 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[4, 0, 23285, 3, 32761, 6829, 2654, 0] avg=8192.0000 max=32761 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[7784, 0, 4, 0, 24980, 32764, 4, 0] avg=8192.0000 max=32764 max_rank=5 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[8274, 8275, 8275, 7943, 8275, 7950, 8273, 8271] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=79188 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[4, 0, 23285, 3, 32761, 6829, 2654, 0] avg=8192.0000 max=32761 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[4, 0, 23285, 3, 32761, 6829, 2654, 0] avg=8192.0000 max=32761 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[7950, 8275, 8273, 8274, 8274, 8272, 8275, 7943] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=82974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[4, 0, 23285, 3, 32761, 6829, 2654, 0] avg=8192.0000 max=32761 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[8274, 8275, 8275, 7943, 8275, 7950, 8273, 8271] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=79188 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[7950, 8275, 8273, 8274, 8274, 8272, 8275, 7943] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=82974 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[4, 0, 4, 25197, 0, 12607, 8, 27716] avg=8192.0000 max=27716 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.383301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[8274, 8275, 8275, 7943, 8275, 7950, 8273, 8271] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=79188 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[8274, 8275, 8275, 7943, 8275, 7950, 8273, 8271] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=79188 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[4, 0, 4, 25197, 0, 12607, 8, 27716] avg=8192.0000 max=27716 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.383301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[8274, 8275, 8275, 7943, 8275, 7950, 8273, 8271] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=79188 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[4, 0, 23285, 3, 32761, 6829, 2654, 0] avg=8192.0000 max=32761 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[12594, 12607, 4, 12607, 2502, 12607, 8, 12607] avg=8192.0000 max=12607 max_rank=1 min=4 min_rank=2 imbalance_ratio=1.538940 max_load_reduction=15109 max_load_reduction_pct=0.545136 moved_to_replicas=65243 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[4, 0, 4, 25197, 0, 12607, 8, 27716] avg=8192.0000 max=27716 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.383301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[4, 0, 23285, 3, 32761, 6829, 2654, 0] avg=8192.0000 max=32761 max_rank=4 min=0 min_rank=1 imbalance_ratio=3.999146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[12594, 12607, 4, 12607, 2502, 12607, 8, 12607] avg=8192.0000 max=12607 max_rank=1 min=4 min_rank=2 imbalance_ratio=1.538940 max_load_reduction=15109 max_load_reduction_pct=0.545136 moved_to_replicas=65243 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[4, 0, 4, 25197, 0, 12607, 8, 27716] avg=8192.0000 max=27716 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.383301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[4, 0, 4, 25197, 0, 12607, 8, 27716] avg=8192.0000 max=27716 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.383301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[8274, 8275, 8275, 7943, 8275, 7950, 8273, 8271] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=79188 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[4, 0, 4, 25197, 0, 12607, 8, 27716] avg=8192.0000 max=27716 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.383301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[12594, 12607, 4, 12607, 2502, 12607, 8, 12607] avg=8192.0000 max=12607 max_rank=1 min=4 min_rank=2 imbalance_ratio=1.538940 max_load_reduction=15109 max_load_reduction_pct=0.545136 moved_to_replicas=65243 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[32552, 0, 32764, 4, 212, 0, 4, 0] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[8274, 8275, 8275, 7943, 8275, 7950, 8273, 8271] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24486 max_load_reduction_pct=0.747413 moved_to_replicas=79188 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[12594, 12607, 4, 12607, 2502, 12607, 8, 12607] avg=8192.0000 max=12607 max_rank=1 min=4 min_rank=2 imbalance_ratio=1.538940 max_load_reduction=15109 max_load_reduction_pct=0.545136 moved_to_replicas=65243 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[12594, 12607, 4, 12607, 2502, 12607, 8, 12607] avg=8192.0000 max=12607 max_rank=1 min=4 min_rank=2 imbalance_ratio=1.538940 max_load_reduction=15109 max_load_reduction_pct=0.545136 moved_to_replicas=65243 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[12594, 12607, 4, 12607, 2502, 12607, 8, 12607] avg=8192.0000 max=12607 max_rank=1 min=4 min_rank=2 imbalance_ratio=1.538940 max_load_reduction=15109 max_load_reduction_pct=0.545136 moved_to_replicas=65243 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[32552, 0, 32764, 4, 212, 0, 4, 0] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7951, 8272, 8273, 7943] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97512 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[4, 0, 4, 25197, 0, 12607, 8, 27716] avg=8192.0000 max=27716 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.383301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[32552, 0, 32764, 4, 212, 0, 4, 0] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[4, 0, 4, 25197, 0, 12607, 8, 27716] avg=8192.0000 max=27716 max_rank=7 min=0 min_rank=1 imbalance_ratio=3.383301 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7951, 8272, 8273, 7943] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97512 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[32552, 0, 32764, 4, 212, 0, 4, 0] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[32552, 0, 32764, 4, 212, 0, 4, 0] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[12594, 12607, 4, 12607, 2502, 12607, 8, 12607] avg=8192.0000 max=12607 max_rank=1 min=4 min_rank=2 imbalance_ratio=1.538940 max_load_reduction=15109 max_load_reduction_pct=0.545136 moved_to_replicas=65243 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[32552, 0, 32764, 4, 212, 0, 4, 0] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[32764, 2, 4, 106, 32660, 0, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=5 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7951, 8272, 8273, 7943] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97512 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[12594, 12607, 4, 12607, 2502, 12607, 8, 12607] avg=8192.0000 max=12607 max_rank=1 min=4 min_rank=2 imbalance_ratio=1.538940 max_load_reduction=15109 max_load_reduction_pct=0.545136 moved_to_replicas=65243 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7951, 8272, 8273, 7943] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97512 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7951, 8272, 8273, 7943] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97512 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7951, 8272, 8273, 7943] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97512 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[32764, 2, 4, 106, 32660, 0, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=5 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[8275, 8274, 8274, 7950, 8274, 8272, 8274, 7943] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97544 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[32552, 0, 32764, 4, 212, 0, 4, 0] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[32764, 2, 4, 106, 32660, 0, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=5 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[32552, 0, 32764, 4, 212, 0, 4, 0] avg=8192.0000 max=32764 max_rank=2 min=0 min_rank=1 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[8275, 8274, 8274, 7950, 8274, 8272, 8274, 7943] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97544 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[32764, 2, 4, 106, 32660, 0, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=5 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[32764, 2, 4, 106, 32660, 0, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=5 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7951, 8272, 8273, 7943] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97512 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[32764, 2, 4, 106, 32660, 0, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=5 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[0, 4, 0, 3, 28, 4, 32733, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[8275, 8274, 8274, 7950, 8274, 8272, 8274, 7943] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97544 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[8274, 8275, 8274, 8274, 7951, 8272, 8273, 7943] avg=8192.0000 max=8275 max_rank=1 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97512 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[8275, 8274, 8274, 7950, 8274, 8272, 8274, 7943] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97544 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[8275, 8274, 8274, 7950, 8274, 8272, 8274, 7943] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97544 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[8275, 8274, 8274, 7950, 8274, 8272, 8274, 7943] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97544 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[0, 4, 0, 3, 28, 4, 32733, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[8275, 8274, 8274, 7943, 7946, 8274, 8275, 8275] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97872 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[32764, 2, 4, 106, 32660, 0, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=5 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[0, 4, 0, 3, 28, 4, 32733, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[32764, 2, 4, 106, 32660, 0, 0, 0] avg=8192.0000 max=32764 max_rank=0 min=0 min_rank=5 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[8275, 8274, 8274, 7943, 7946, 8274, 8275, 8275] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97872 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[0, 4, 0, 3, 28, 4, 32733, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[0, 4, 0, 3, 28, 4, 32733, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[0, 4, 0, 3, 28, 4, 32733, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[8275, 8274, 8274, 7950, 8274, 8272, 8274, 7943] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97544 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[4, 7, 0, 0, 32764, 1, 3, 32757] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[8275, 8274, 8274, 7943, 7946, 8274, 8275, 8275] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97872 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[8275, 8274, 8274, 7950, 8274, 8272, 8274, 7943] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=7 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97544 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[8275, 8274, 8274, 7943, 7946, 8274, 8275, 8275] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97872 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[4, 7, 0, 0, 32764, 1, 3, 32757] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[8275, 8274, 8274, 7943, 7946, 8274, 8275, 8275] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97872 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[8275, 8274, 8274, 7943, 7946, 8274, 8275, 8275] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97872 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[8274, 7950, 8276, 8276, 8272, 7941, 8274, 8273] avg=8192.0000 max=8276 max_rank=2 min=7941 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[0, 4, 0, 3, 28, 4, 32733, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[4, 7, 0, 0, 32764, 1, 3, 32757] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[0, 4, 0, 3, 28, 4, 32733, 32764] avg=8192.0000 max=32764 max_rank=7 min=0 min_rank=0 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[8274, 7950, 8276, 8276, 8272, 7941, 8274, 8273] avg=8192.0000 max=8276 max_rank=2 min=7941 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[4, 7, 0, 0, 32764, 1, 3, 32757] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[4, 7, 0, 0, 32764, 1, 3, 32757] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[4, 7, 0, 0, 32764, 1, 3, 32757] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[8275, 8274, 8274, 7943, 7946, 8274, 8275, 8275] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97872 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[8274, 7950, 8276, 8276, 8272, 7941, 8274, 8273] avg=8192.0000 max=8276 max_rank=2 min=7941 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[8275, 8274, 8274, 7943, 7946, 8274, 8275, 8275] avg=8192.0000 max=8275 max_rank=0 min=7943 min_rank=3 imbalance_ratio=1.010132 max_load_reduction=24489 max_load_reduction_pct=0.747436 moved_to_replicas=97872 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[8274, 7950, 8276, 8276, 8272, 7941, 8274, 8273] avg=8192.0000 max=8276 max_rank=2 min=7941 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[8274, 7950, 8276, 8276, 8272, 7941, 8274, 8273] avg=8192.0000 max=8276 max_rank=2 min=7941 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[8274, 7950, 8276, 8276, 8272, 7941, 8274, 8273] avg=8192.0000 max=8276 max_rank=2 min=7941 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[4, 7, 0, 0, 32764, 1, 3, 32757] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[4, 7, 0, 0, 32764, 1, 3, 32757] avg=8192.0000 max=32764 max_rank=4 min=0 min_rank=2 imbalance_ratio=3.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[8274, 7950, 8276, 8276, 8272, 7941, 8274, 8273] avg=8192.0000 max=8276 max_rank=2 min=7941 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[8274, 7950, 8276, 8276, 8272, 7941, 8274, 8273] avg=8192.0000 max=8276 max_rank=2 min=7941 min_rank=5 imbalance_ratio=1.010254 max_load_reduction=24488 max_load_reduction_pct=0.747406 moved_to_replicas=97926 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=32 total_expected_recv=278550 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=193024 peak_demand=14380 required_capacity=1.755371 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=32 total_expected_recv=268724 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=203136 peak_demand=14380 required_capacity=1.755371 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=32 total_expected_recv=248589 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=223488 peak_demand=9313 required_capacity=1.136841 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=32 total_expected_recv=275220 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=196352 peak_demand=13064 required_capacity=1.594727 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=32 total_expected_recv=263602 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=208384 peak_demand=13887 required_capacity=1.695190 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=32 total_expected_recv=260788 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=210048 peak_demand=12594 required_capacity=1.537354 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=32 total_expected_recv=255187 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=216704 peak_demand=12607 required_capacity=1.538940 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=32 total_expected_recv=246492 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=224640 peak_demand=12560 required_capacity=1.533203 + [2026-08-30 05:09:21] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 910.6 | throughput per GPU (TFLOP/s/GPU): 366.1 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.308262E+00 | loss scale: 1.0 | grad norm: 5.594 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 14592 | min safe capacity: 1.7812 | +[after training is done] datetime: 2026-08-30 05:09:21 +[rank0]:[W830 05:09:22.796053485 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 05:09:24.009000 481867 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 05:09:24.010000 481867 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 481959 closing signal SIGTERM +W0830 05:09:24.010000 481867 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 481960 closing signal SIGTERM +W0830 05:09:24.011000 481867 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 481961 closing signal SIGTERM +W0830 05:09:24.012000 481867 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 481962 closing signal SIGTERM +W0830 05:09:24.013000 481867 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 481963 closing signal SIGTERM +W0830 05:09:24.014000 481867 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 481964 closing signal SIGTERM +W0830 05:09:24.015000 481867 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 481965 closing signal SIGTERM +W0830 05:09:24.015000 481867 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 481966 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 481867 got signal: 15 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..807b7ecc2c7f7e751399a7b557cd72bc1cadc2b4 --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 629 +4, 37 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1895 +1, 1895 +2, 1895 +3, 1895 +4, 1895 +5, 1895 +6, 1895 +7, 1895 +0, 2399 +1, 2389 +2, 2401 +3, 2379 +4, 2393 +5, 2399 +6, 2399 +7, 2391 +0, 42337 +1, 42333 +2, 42333 +3, 42333 +4, 42333 +5, 42333 +6, 42333 +7, 42333 +0, 42743 +1, 42733 +2, 42733 +3, 42733 +4, 42733 +5, 42733 +6, 42733 +7, 42733 +0, 44049 +1, 44041 +2, 44041 +3, 44041 +4, 44041 +5, 44041 +6, 44041 +7, 44041 +0, 44321 +1, 44313 +2, 44313 +3, 44313 +4, 44313 +5, 44313 +6, 44313 +7, 44313 +0, 44321 +1, 44313 +2, 44313 +3, 44313 +4, 44313 +5, 44313 +6, 44313 +7, 44313 +0, 45721 +1, 45713 +2, 45713 +3, 45713 +4, 45713 +5, 45713 +6, 45713 +7, 45713 +0, 62317 +1, 62311 +2, 62311 +3, 62309 +4, 62313 +5, 62037 +6, 62313 +7, 62311 +0, 70515 +1, 70563 +2, 70563 +3, 70563 +4, 70521 +5, 70563 +6, 70563 +7, 70563 +0, 70991 +1, 71039 +2, 71039 +3, 71039 +4, 70997 +5, 71039 +6, 71039 +7, 71039 +0, 70991 +1, 71039 +2, 71039 +3, 71039 +4, 70997 +5, 71039 +6, 71039 +7, 71039 +0, 82919 +1, 82957 +2, 82959 +3, 82975 +4, 82967 +5, 82963 +6, 82985 +7, 82997 +0, 83857 +1, 83921 +2, 83939 +3, 83933 +4, 83991 +5, 83923 +6, 83957 +7, 83937 +0, 84961 +1, 85043 +2, 85027 +3, 85045 +4, 85047 +5, 85033 +6, 85059 +7, 85043 +0, 107987 +1, 107977 +2, 107977 +3, 107977 +4, 107915 +5, 107977 +6, 107977 +7, 107977 +0, 107987 +1, 107977 +2, 107977 +3, 107977 +4, 107915 +5, 107977 +6, 107977 +7, 107977 +0, 107987 +1, 107977 +2, 107977 +3, 107977 +4, 107915 +5, 107977 +6, 107977 +7, 107977 +0, 107071 +1, 107239 +2, 107123 +3, 107323 +4, 107309 +5, 107061 +6, 107061 +7, 107231 +0, 103143 +1, 103201 +2, 102813 +3, 102813 +4, 102815 +5, 103459 +6, 103197 +7, 103459 +0, 100755 +1, 101197 +2, 101779 +3, 101453 +4, 101071 +5, 100745 +6, 101133 +7, 101133 +0, 1 +1, 65 +2, 101746 +3, 321 +4, 4 +5, 4 +6, 4 +7, 4 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/run.env b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..fdb517ac5371b158e4bb4d12c2f371545618b7da --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/run.env @@ -0,0 +1,70 @@ +backend=ultraep +config=G2 +family=tierA +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=3584 +num_experts=32 +topk=8 +experts_per_rank=4 +expert_params=44040192 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=32768 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=4 +capacity_mode=static +capacity=1.0625 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=1024 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/train.log b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..4fd133c841b2a137b05f5a3e2614618cb45fe72b --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/train.log @@ -0,0 +1,8955 @@ +[05:10:43] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + G2 / ultraep (slots/rank=4 policy=matched) + E=32 K=8 H'=3584 m=4 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=static cap=1.0625 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0830 05:10:44.377000 488791 torch/distributed/run.py:766] +W0830 05:10:44.377000 488791 torch/distributed/run.py:766] ***************************************** +W0830 05:10:44.377000 488791 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 05:10:44.377000 488791 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 3584 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 4 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.0625 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 32 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 67 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W830 05:11:02.122569760 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 05:11:02.122649668 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 05:11:02.122843346 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 05:11:02.123509158 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 05:11:02.123654473 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 05:11:02.123675306 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 05:11:02.123880403 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 05:11:03.299744932 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.840 seconds +time to initialize megatron (seconds): 7.268 +[after megatron is initialized] datetime: 2026-08-30 05:11:08 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 6722555904 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (42086400 elements, 42086400 padded size): + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.final_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight +Params for bucket 3 (42082304 elements, 42082304 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.pre_mlp_layernorm.weight +Params for bucket 4 (42082304 elements, 42082304 padded size): + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.linear_qkv.weight +Params for bucket 5 (42082304 elements, 42082304 padded size): + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.linear_qkv.weight +Params for bucket 6 (42082304 elements, 42082304 padded size): + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight +Params for bucket 7 (42082304 elements, 42082304 padded size): + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_proj.weight +Params for bucket 8 (42082304 elements, 42082304 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.weight +Params for bucket 9 (42082304 elements, 42082304 padded size): + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight +Params for bucket 10 (42082304 elements, 42082304 padded size): + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_qkv.weight +Params for bucket 11 (42082304 elements, 42082304 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.weight +Params for bucket 12 (42082304 elements, 42082304 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.pre_mlp_layernorm.weight +Params for bucket 13 (42082304 elements, 42082304 padded size): + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 14 (42082304 elements, 42082304 padded size): + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 15 (42082304 elements, 42082304 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.pre_mlp_layernorm.weight +Params for bucket 16 (42082304 elements, 42082304 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.pre_mlp_layernorm.weight +Params for bucket 17 (42082304 elements, 42082304 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 56 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 +Params for bucket 3 (58720256 elements, 58720256 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 +Params for bucket 5 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 +Params for bucket 6 (58720256 elements, 58720256 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 +Params for bucket 7 (58720256 elements, 58720256 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 +Params for bucket 9 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 +Params for bucket 10 (58720256 elements, 58720256 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 +Params for bucket 13 (58720256 elements, 58720256 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 +Params for bucket 14 (58720256 elements, 58720256 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 17 (58720256 elements, 58720256 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 +Params for bucket 18 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 +Params for bucket 20 (58720256 elements, 58720256 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 +Params for bucket 21 (58720256 elements, 58720256 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 +Params for bucket 23 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 +Params for bucket 24 (58720256 elements, 58720256 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 +Params for bucket 27 (58720256 elements, 58720256 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 +Params for bucket 28 (58720256 elements, 58720256 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 29 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 +Params for bucket 30 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 31 (58720256 elements, 58720256 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 +Params for bucket 34 (58720256 elements, 58720256 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 +Params for bucket 35 (58720256 elements, 58720256 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 +Params for bucket 36 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 +Params for bucket 37 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 38 (58720256 elements, 58720256 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 +Params for bucket 41 (58720256 elements, 58720256 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 +Params for bucket 42 (58720256 elements, 58720256 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 43 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 +Params for bucket 44 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 +Params for bucket 45 (58720256 elements, 58720256 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 +Params for bucket 47 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 +Params for bucket 48 (58720256 elements, 58720256 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 +Params for bucket 49 (58720256 elements, 58720256 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 50 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 52 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +Params for bucket 53 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 +Params for bucket 54 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 +Params for bucket 55 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 +Params for bucket 56 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 05:11:12 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 192 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 05:11:12 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3196.66, 3200.72) + train/valid/test-data-iterators-setup ..........: (73.46, 90.64) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-30 05:11:12 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=34816 expected_recv=66152 expected_overflow=0 exact_recv_per_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=34816 expected_recv=66152 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=34816 expected_recv=63808 expected_overflow=0 exact_recv_per_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=34816 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=34816 expected_recv=63808 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=64512 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=34816 expected_recv=66190 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=34816 expected_recv=64825 expected_overflow=0 exact_recv_per_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=34816 expected_recv=64282 expected_overflow=0 exact_recv_per_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=34816 expected_recv=64825 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=34816 expected_recv=64282 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=34816 expected_recv=65381 expected_overflow=0 exact_recv_per_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=64640 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=34816 expected_recv=65381 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=34816 expected_recv=64596 expected_overflow=0 exact_recv_per_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=34816 expected_recv=65733 expected_overflow=0 exact_recv_per_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=34816 expected_recv=64596 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=34816 expected_recv=65733 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=34816 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=34816 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=34816 expected_recv=64681 expected_overflow=0 exact_recv_per_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=34816 expected_recv=66194 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=34816 expected_recv=66194 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=34816 expected_recv=64681 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=34816 expected_recv=64374 expected_overflow=0 exact_recv_per_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=34816 expected_recv=64374 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=34816 expected_recv=65307 expected_overflow=0 exact_recv_per_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=64896 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=34816 expected_recv=65307 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=34816 expected_recv=65718 expected_overflow=0 exact_recv_per_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=34816 expected_recv=65385 expected_overflow=0 exact_recv_per_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=34816 expected_recv=65718 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=34816 expected_recv=65385 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=34816 expected_recv=65719 expected_overflow=0 exact_recv_per_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=34816 expected_recv=65719 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=34816 expected_recv=65695 expected_overflow=0 exact_recv_per_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=34816 expected_recv=66037 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=34816 expected_recv=65437 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=34816 expected_recv=65695 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=34816 expected_recv=66037 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=34816 expected_recv=64042 expected_overflow=0 exact_recv_per_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=34816 expected_recv=65437 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=34816 expected_recv=64042 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=64512 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=34816 expected_recv=64639 expected_overflow=0 exact_recv_per_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=34816 expected_recv=64639 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=34816 expected_recv=64731 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=34816 expected_recv=64731 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=34816 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=34816 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=34816 expected_recv=65539 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=34816 expected_recv=63568 actual_valid_recv=64000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=64000 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=34816 expected_recv=65191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=34816 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=34816 expected_recv=65191 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=34816 expected_recv=63568 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=64256 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=34816 expected_recv=63722 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=34816 expected_recv=63722 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=64256 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=34816 expected_recv=64263 expected_overflow=0 exact_recv_per_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=34816 expected_recv=65129 expected_overflow=0 exact_recv_per_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=34816 expected_recv=64433 expected_overflow=0 exact_recv_per_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=34816 expected_recv=65914 expected_overflow=0 exact_recv_per_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=34816 expected_recv=65163 expected_overflow=0 exact_recv_per_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=34816 expected_recv=65129 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=34816 expected_recv=64433 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=34816 expected_recv=65914 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=34816 expected_recv=65163 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=34816 expected_recv=64263 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=34816 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=34816 expected_recv=65508 expected_overflow=0 exact_recv_per_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=34816 expected_recv=66194 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=34816 expected_recv=65508 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=34816 expected_recv=65964 expected_overflow=0 exact_recv_per_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=34816 expected_recv=65964 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=34816 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=34816 expected_recv=66190 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=34816 expected_recv=65491 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=34816 expected_recv=65491 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=34816 expected_recv=62763 expected_overflow=0 exact_recv_per_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=34816 expected_recv=64213 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=34816 expected_recv=65401 expected_overflow=0 exact_recv_per_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=34816 expected_recv=62763 actual_valid_recv=63616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=34816 expected_recv=64213 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=34816 expected_recv=65401 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=63616 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=64640 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=34816 expected_recv=64537 expected_overflow=0 exact_recv_per_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=34816 expected_recv=64192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=34816 expected_recv=64537 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=34816 expected_recv=64192 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=34816 expected_recv=65583 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=34816 expected_recv=64664 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=64768 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=34816 expected_recv=65583 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=34816 expected_recv=65057 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=34816 expected_recv=64664 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=34816 expected_recv=65569 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=34816 expected_recv=65057 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=34816 expected_recv=65569 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=34816 expected_recv=65137 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=34816 expected_recv=65137 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=34816 expected_recv=64795 expected_overflow=0 exact_recv_per_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=34816 expected_recv=64557 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=34816 expected_recv=64795 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=34816 expected_recv=64878 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=34816 expected_recv=64557 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=34816 expected_recv=64569 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=34816 expected_recv=64878 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65024 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=34816 expected_recv=65204 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=34816 expected_recv=64569 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=34816 expected_recv=65204 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=34816 expected_recv=63567 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=34816 expected_recv=63567 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=64512 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=34816 expected_recv=63750 expected_overflow=0 exact_recv_per_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=34816 expected_recv=63750 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 26996, 22461, 22979, 50624, 30904, 43374] avg=32768.0000 max=50624 max_rank=5 min=22461 min_rank=3 imbalance_ratio=1.544922 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=64512 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=34816 expected_recv=65320 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=34816 expected_recv=65320 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] avg=32768.0000 max=33097 max_rank=3 min=31750 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=17527 max_load_reduction_pct=0.346219 moved_to_replicas=58431 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=34816 expected_recv=64967 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=34816 expected_recv=64967 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=34816 expected_recv=64175 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=34816 expected_recv=63839 expected_overflow=0 exact_recv_per_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=34816 expected_recv=64175 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[16668, 73537, 11299, 39557, 40193, 2514, 45521, 32855] avg=32768.0000 max=73537 max_rank=1 min=2514 min_rank=5 imbalance_ratio=2.244171 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=34816 expected_recv=63839 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=64896 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=34816 expected_recv=65075 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=64256 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=34816 expected_recv=65075 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 26996, 22461, 22979, 50624, 30904, 43374] avg=32768.0000 max=50624 max_rank=5 min=22461 min_rank=3 imbalance_ratio=1.544922 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=34816 expected_recv=64684 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] avg=32768.0000 max=33097 max_rank=1 min=30712 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40440 max_load_reduction_pct=0.549927 moved_to_replicas=117254 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=34816 expected_recv=64684 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=34816 expected_recv=64883 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=34816 expected_recv=64883 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] avg=32768.0000 max=33097 max_rank=3 min=31750 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=17527 max_load_reduction_pct=0.346219 moved_to_replicas=58431 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=34816 expected_recv=64037 expected_overflow=0 exact_recv_per_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=34816 expected_recv=64037 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[58778, 32211, 41142, 29026, 53735, 18463, 7817, 20972] avg=32768.0000 max=58778 max_rank=0 min=7817 min_rank=6 imbalance_ratio=1.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=64896 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 26996, 22461, 22979, 50624, 30904, 43374] avg=32768.0000 max=50624 max_rank=5 min=22461 min_rank=3 imbalance_ratio=1.544922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[16668, 73537, 11299, 39557, 40193, 2514, 45521, 32855] avg=32768.0000 max=73537 max_rank=1 min=2514 min_rank=5 imbalance_ratio=2.244171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31585 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25681 max_load_reduction_pct=0.436915 moved_to_replicas=96118 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 26996, 22461, 22979, 50624, 30904, 43374] avg=32768.0000 max=50624 max_rank=5 min=22461 min_rank=3 imbalance_ratio=1.544922 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 26996, 22461, 22979, 50624, 30904, 43374] avg=32768.0000 max=50624 max_rank=5 min=22461 min_rank=3 imbalance_ratio=1.544922 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=34816 expected_recv=64879 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] avg=32768.0000 max=33097 max_rank=3 min=31750 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=17527 max_load_reduction_pct=0.346219 moved_to_replicas=58431 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] avg=32768.0000 max=33097 max_rank=1 min=30712 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40440 max_load_reduction_pct=0.549927 moved_to_replicas=117254 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=34816 expected_recv=64879 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 26996, 22461, 22979, 50624, 30904, 43374] avg=32768.0000 max=50624 max_rank=5 min=22461 min_rank=3 imbalance_ratio=1.544922 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] avg=32768.0000 max=33097 max_rank=3 min=31750 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=17527 max_load_reduction_pct=0.346219 moved_to_replicas=58431 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[39478, 24583, 37640, 47248, 26898, 21200, 41111, 23986] avg=32768.0000 max=47248 max_rank=3 min=21200 min_rank=5 imbalance_ratio=1.441895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] avg=32768.0000 max=33097 max_rank=3 min=31750 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=17527 max_load_reduction_pct=0.346219 moved_to_replicas=58431 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=34816 expected_recv=65095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[58778, 32211, 41142, 29026, 53735, 18463, 7817, 20972] avg=32768.0000 max=58778 max_rank=0 min=7817 min_rank=6 imbalance_ratio=1.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] avg=32768.0000 max=33096 max_rank=0 min=31442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14152 max_load_reduction_pct=0.299526 moved_to_replicas=73219 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] avg=32768.0000 max=33097 max_rank=3 min=31750 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=17527 max_load_reduction_pct=0.346219 moved_to_replicas=58431 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[16668, 73537, 11299, 39557, 40193, 2514, 45521, 32855] avg=32768.0000 max=73537 max_rank=1 min=2514 min_rank=5 imbalance_ratio=2.244171 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=34816 expected_recv=65095 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[16668, 73537, 11299, 39557, 40193, 2514, 45521, 32855] avg=32768.0000 max=73537 max_rank=1 min=2514 min_rank=5 imbalance_ratio=2.244171 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[16668, 73537, 11299, 39557, 40193, 2514, 45521, 32855] avg=32768.0000 max=73537 max_rank=1 min=2514 min_rank=5 imbalance_ratio=2.244171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31585 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25681 max_load_reduction_pct=0.436915 moved_to_replicas=96118 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] avg=32768.0000 max=33097 max_rank=1 min=30712 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40440 max_load_reduction_pct=0.549927 moved_to_replicas=117254 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[43778, 33972, 41269, 27420, 22573, 31464, 23303, 38365] avg=32768.0000 max=43778 max_rank=0 min=22573 min_rank=4 imbalance_ratio=1.335999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] avg=32768.0000 max=33097 max_rank=1 min=30712 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40440 max_load_reduction_pct=0.549927 moved_to_replicas=117254 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[16668, 73537, 11299, 39557, 40193, 2514, 45521, 32855] avg=32768.0000 max=73537 max_rank=1 min=2514 min_rank=5 imbalance_ratio=2.244171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] avg=32768.0000 max=33097 max_rank=1 min=30712 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40440 max_load_reduction_pct=0.549927 moved_to_replicas=117254 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[39478, 24583, 37640, 47248, 26898, 21200, 41111, 23986] avg=32768.0000 max=47248 max_rank=3 min=21200 min_rank=5 imbalance_ratio=1.441895 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] avg=32768.0000 max=33098 max_rank=0 min=31634 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=10680 max_load_reduction_pct=0.243958 moved_to_replicas=61433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[58778, 32211, 41142, 29026, 53735, 18463, 7817, 20972] avg=32768.0000 max=58778 max_rank=0 min=7817 min_rank=6 imbalance_ratio=1.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] avg=32768.0000 max=33097 max_rank=1 min=30712 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40440 max_load_reduction_pct=0.549927 moved_to_replicas=117254 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[58778, 32211, 41142, 29026, 53735, 18463, 7817, 20972] avg=32768.0000 max=58778 max_rank=0 min=7817 min_rank=6 imbalance_ratio=1.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] avg=32768.0000 max=33096 max_rank=0 min=31442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14152 max_load_reduction_pct=0.299526 moved_to_replicas=73219 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[58778, 32211, 41142, 29026, 53735, 18463, 7817, 20972] avg=32768.0000 max=58778 max_rank=0 min=7817 min_rank=6 imbalance_ratio=1.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31585 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25681 max_load_reduction_pct=0.436915 moved_to_replicas=96118 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[12266, 35078, 70503, 54837, 12318, 10254, 49173, 17715] avg=32768.0000 max=70503 max_rank=2 min=10254 min_rank=5 imbalance_ratio=2.151581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31585 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25681 max_load_reduction_pct=0.436915 moved_to_replicas=96118 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31585 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25681 max_load_reduction_pct=0.436915 moved_to_replicas=96118 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[58778, 32211, 41142, 29026, 53735, 18463, 7817, 20972] avg=32768.0000 max=58778 max_rank=0 min=7817 min_rank=6 imbalance_ratio=1.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[43778, 33972, 41269, 27420, 22573, 31464, 23303, 38365] avg=32768.0000 max=43778 max_rank=0 min=22573 min_rank=4 imbalance_ratio=1.335999 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=30471 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37406 max_load_reduction_pct=0.530559 moved_to_replicas=150937 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[39478, 24583, 37640, 47248, 26898, 21200, 41111, 23986] avg=32768.0000 max=47248 max_rank=3 min=21200 min_rank=5 imbalance_ratio=1.441895 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31585 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25681 max_load_reduction_pct=0.436915 moved_to_replicas=96118 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[39478, 24583, 37640, 47248, 26898, 21200, 41111, 23986] avg=32768.0000 max=47248 max_rank=3 min=21200 min_rank=5 imbalance_ratio=1.441895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] avg=32768.0000 max=33098 max_rank=0 min=31634 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=10680 max_load_reduction_pct=0.243958 moved_to_replicas=61433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[39478, 24583, 37640, 47248, 26898, 21200, 41111, 23986] avg=32768.0000 max=47248 max_rank=3 min=21200 min_rank=5 imbalance_ratio=1.441895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[41103, 69582, 53793, 28787, 22039, 14246, 25077, 7517] avg=32768.0000 max=69582 max_rank=1 min=7517 min_rank=7 imbalance_ratio=2.123474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] avg=32768.0000 max=33096 max_rank=0 min=31442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14152 max_load_reduction_pct=0.299526 moved_to_replicas=73219 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] avg=32768.0000 max=33096 max_rank=0 min=31442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14152 max_load_reduction_pct=0.299526 moved_to_replicas=73219 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=34816 expected_recv=64784 expected_overflow=0 exact_recv_per_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] avg=32768.0000 max=33096 max_rank=0 min=31442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14152 max_load_reduction_pct=0.299526 moved_to_replicas=73219 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[39478, 24583, 37640, 47248, 26898, 21200, 41111, 23986] avg=32768.0000 max=47248 max_rank=3 min=21200 min_rank=5 imbalance_ratio=1.441895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[12266, 35078, 70503, 54837, 12318, 10254, 49173, 17715] avg=32768.0000 max=70503 max_rank=2 min=10254 min_rank=5 imbalance_ratio=2.151581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] avg=32768.0000 max=33096 max_rank=0 min=31473 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36486 max_load_reduction_pct=0.524360 moved_to_replicas=113366 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=34816 expected_recv=64784 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[43778, 33972, 41269, 27420, 22573, 31464, 23303, 38365] avg=32768.0000 max=43778 max_rank=0 min=22573 min_rank=4 imbalance_ratio=1.335999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] avg=32768.0000 max=33096 max_rank=0 min=31442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14152 max_load_reduction_pct=0.299526 moved_to_replicas=73219 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=30471 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37406 max_load_reduction_pct=0.530559 moved_to_replicas=150937 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[43778, 33972, 41269, 27420, 22573, 31464, 23303, 38365] avg=32768.0000 max=43778 max_rank=0 min=22573 min_rank=4 imbalance_ratio=1.335999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[43778, 33972, 41269, 27420, 22573, 31464, 23303, 38365] avg=32768.0000 max=43778 max_rank=0 min=22573 min_rank=4 imbalance_ratio=1.335999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[17432, 34096, 41175, 38016, 25559, 49545, 39563, 16758] avg=32768.0000 max=49545 max_rank=5 min=16758 min_rank=7 imbalance_ratio=1.511993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] avg=32768.0000 max=33098 max_rank=0 min=31634 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=10680 max_load_reduction_pct=0.243958 moved_to_replicas=61433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] avg=32768.0000 max=33098 max_rank=0 min=31634 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=10680 max_load_reduction_pct=0.243958 moved_to_replicas=61433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] avg=32768.0000 max=33098 max_rank=0 min=31634 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=10680 max_load_reduction_pct=0.243958 moved_to_replicas=61433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[41103, 69582, 53793, 28787, 22039, 14246, 25077, 7517] avg=32768.0000 max=69582 max_rank=1 min=7517 min_rank=7 imbalance_ratio=2.123474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31542 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16449 max_load_reduction_pct=0.332001 moved_to_replicas=82564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[43778, 33972, 41269, 27420, 22573, 31464, 23303, 38365] avg=32768.0000 max=43778 max_rank=0 min=22573 min_rank=4 imbalance_ratio=1.335999 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[12266, 35078, 70503, 54837, 12318, 10254, 49173, 17715] avg=32768.0000 max=70503 max_rank=2 min=10254 min_rank=5 imbalance_ratio=2.151581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] avg=32768.0000 max=33096 max_rank=0 min=31473 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36486 max_load_reduction_pct=0.524360 moved_to_replicas=113366 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] avg=32768.0000 max=33098 max_rank=0 min=31634 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=10680 max_load_reduction_pct=0.243958 moved_to_replicas=61433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[12266, 35078, 70503, 54837, 12318, 10254, 49173, 17715] avg=32768.0000 max=70503 max_rank=2 min=10254 min_rank=5 imbalance_ratio=2.151581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[24024, 37006, 47284, 42347, 21548, 15751, 36397, 37787] avg=32768.0000 max=47284 max_rank=2 min=15751 min_rank=5 imbalance_ratio=1.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[12266, 35078, 70503, 54837, 12318, 10254, 49173, 17715] avg=32768.0000 max=70503 max_rank=2 min=10254 min_rank=5 imbalance_ratio=2.151581 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=34816 expected_recv=63569 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=30471 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37406 max_load_reduction_pct=0.530559 moved_to_replicas=150937 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=34816 expected_recv=63569 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=64256 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[17432, 34096, 41175, 38016, 25559, 49545, 39563, 16758] avg=32768.0000 max=49545 max_rank=5 min=16758 min_rank=7 imbalance_ratio=1.511993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 26996, 22461, 22979, 50624, 30904, 43374] avg=32768.0000 max=50624 max_rank=5 min=22461 min_rank=3 imbalance_ratio=1.544922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30942 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14187 max_load_reduction_pct=0.300038 moved_to_replicas=56241 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=30471 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37406 max_load_reduction_pct=0.530559 moved_to_replicas=150937 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=30471 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37406 max_load_reduction_pct=0.530559 moved_to_replicas=150937 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[12266, 35078, 70503, 54837, 12318, 10254, 49173, 17715] avg=32768.0000 max=70503 max_rank=2 min=10254 min_rank=5 imbalance_ratio=2.151581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[41103, 69582, 53793, 28787, 22039, 14246, 25077, 7517] avg=32768.0000 max=69582 max_rank=1 min=7517 min_rank=7 imbalance_ratio=2.123474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31542 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16449 max_load_reduction_pct=0.332001 moved_to_replicas=82564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] avg=32768.0000 max=33097 max_rank=3 min=31750 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=17527 max_load_reduction_pct=0.346219 moved_to_replicas=58431 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=30471 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37406 max_load_reduction_pct=0.530559 moved_to_replicas=150937 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[12561, 22511, 50842, 22324, 21257, 41284, 45703, 45662] avg=32768.0000 max=50842 max_rank=2 min=12561 min_rank=0 imbalance_ratio=1.551575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] avg=32768.0000 max=33096 max_rank=0 min=31473 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36486 max_load_reduction_pct=0.524360 moved_to_replicas=113366 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[41103, 69582, 53793, 28787, 22039, 14246, 25077, 7517] avg=32768.0000 max=69582 max_rank=1 min=7517 min_rank=7 imbalance_ratio=2.123474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[41103, 69582, 53793, 28787, 22039, 14246, 25077, 7517] avg=32768.0000 max=69582 max_rank=1 min=7517 min_rank=7 imbalance_ratio=2.123474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[24024, 37006, 47284, 42347, 21548, 15751, 36397, 37787] avg=32768.0000 max=47284 max_rank=2 min=15751 min_rank=5 imbalance_ratio=1.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 26996, 22461, 22979, 50624, 30904, 43374] avg=32768.0000 max=50624 max_rank=5 min=22461 min_rank=3 imbalance_ratio=1.544922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17745 max_load_reduction_pct=0.349022 moved_to_replicas=71494 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[16668, 73537, 11299, 39557, 40193, 2514, 45521, 32855] avg=32768.0000 max=73537 max_rank=1 min=2514 min_rank=5 imbalance_ratio=2.244171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] avg=32768.0000 max=33096 max_rank=0 min=31473 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36486 max_load_reduction_pct=0.524360 moved_to_replicas=113366 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] avg=32768.0000 max=33096 max_rank=0 min=31473 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36486 max_load_reduction_pct=0.524360 moved_to_replicas=113366 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[41103, 69582, 53793, 28787, 22039, 14246, 25077, 7517] avg=32768.0000 max=69582 max_rank=1 min=7517 min_rank=7 imbalance_ratio=2.123474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30942 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14187 max_load_reduction_pct=0.300038 moved_to_replicas=56241 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[17432, 34096, 41175, 38016, 25559, 49545, 39563, 16758] avg=32768.0000 max=49545 max_rank=5 min=16758 min_rank=7 imbalance_ratio=1.511993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] avg=32768.0000 max=33097 max_rank=3 min=31750 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=17527 max_load_reduction_pct=0.346219 moved_to_replicas=58431 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] avg=32768.0000 max=33097 max_rank=1 min=30712 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40440 max_load_reduction_pct=0.549927 moved_to_replicas=117254 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[8708, 3426, 66610, 15130, 39667, 52261, 20522, 55820] avg=32768.0000 max=66610 max_rank=2 min=3426 min_rank=1 imbalance_ratio=2.032776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] avg=32768.0000 max=33096 max_rank=0 min=31473 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36486 max_load_reduction_pct=0.524360 moved_to_replicas=113366 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31542 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16449 max_load_reduction_pct=0.332001 moved_to_replicas=82564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[17432, 34096, 41175, 38016, 25559, 49545, 39563, 16758] avg=32768.0000 max=49545 max_rank=5 min=16758 min_rank=7 imbalance_ratio=1.511993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[17432, 34096, 41175, 38016, 25559, 49545, 39563, 16758] avg=32768.0000 max=49545 max_rank=5 min=16758 min_rank=7 imbalance_ratio=1.511993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[12561, 22511, 50842, 22324, 21257, 41284, 45703, 45662] avg=32768.0000 max=50842 max_rank=2 min=12561 min_rank=0 imbalance_ratio=1.551575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] avg=32768.0000 max=33097 max_rank=3 min=31263 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33513 max_load_reduction_pct=0.503123 moved_to_replicas=159673 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[16668, 73537, 11299, 39557, 40193, 2514, 45521, 32855] avg=32768.0000 max=73537 max_rank=1 min=2514 min_rank=5 imbalance_ratio=2.244171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31542 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16449 max_load_reduction_pct=0.332001 moved_to_replicas=82564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17745 max_load_reduction_pct=0.349022 moved_to_replicas=71494 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31542 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16449 max_load_reduction_pct=0.332001 moved_to_replicas=82564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[17432, 34096, 41175, 38016, 25559, 49545, 39563, 16758] avg=32768.0000 max=49545 max_rank=5 min=16758 min_rank=7 imbalance_ratio=1.511993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[58778, 32211, 41142, 29026, 53735, 18463, 7817, 20972] avg=32768.0000 max=58778 max_rank=0 min=7817 min_rank=6 imbalance_ratio=1.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[24024, 37006, 47284, 42347, 21548, 15751, 36397, 37787] avg=32768.0000 max=47284 max_rank=2 min=15751 min_rank=5 imbalance_ratio=1.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] avg=32768.0000 max=33097 max_rank=1 min=30712 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40440 max_load_reduction_pct=0.549927 moved_to_replicas=117254 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[35982, 45267, 18285, 37998, 69653, 13599, 24684, 16676] avg=32768.0000 max=69653 max_rank=4 min=13599 min_rank=5 imbalance_ratio=2.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31542 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16449 max_load_reduction_pct=0.332001 moved_to_replicas=82564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31585 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25681 max_load_reduction_pct=0.436915 moved_to_replicas=96118 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30942 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14187 max_load_reduction_pct=0.300038 moved_to_replicas=56241 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[24024, 37006, 47284, 42347, 21548, 15751, 36397, 37787] avg=32768.0000 max=47284 max_rank=2 min=15751 min_rank=5 imbalance_ratio=1.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[8708, 3426, 66610, 15130, 39667, 52261, 20522, 55820] avg=32768.0000 max=66610 max_rank=2 min=3426 min_rank=1 imbalance_ratio=2.032776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[24024, 37006, 47284, 42347, 21548, 15751, 36397, 37787] avg=32768.0000 max=47284 max_rank=2 min=15751 min_rank=5 imbalance_ratio=1.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] avg=32768.0000 max=33096 max_rank=0 min=31096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36557 max_load_reduction_pct=0.524845 moved_to_replicas=103537 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[58778, 32211, 41142, 29026, 53735, 18463, 7817, 20972] avg=32768.0000 max=58778 max_rank=0 min=7817 min_rank=6 imbalance_ratio=1.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] avg=32768.0000 max=33097 max_rank=3 min=31263 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33513 max_load_reduction_pct=0.503123 moved_to_replicas=159673 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30942 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14187 max_load_reduction_pct=0.300038 moved_to_replicas=56241 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30942 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14187 max_load_reduction_pct=0.300038 moved_to_replicas=56241 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[24024, 37006, 47284, 42347, 21548, 15751, 36397, 37787] avg=32768.0000 max=47284 max_rank=2 min=15751 min_rank=5 imbalance_ratio=1.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[39478, 24583, 37640, 47248, 26898, 21200, 41111, 23986] avg=32768.0000 max=47248 max_rank=3 min=21200 min_rank=5 imbalance_ratio=1.441895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[12561, 22511, 50842, 22324, 21257, 41284, 45703, 45662] avg=32768.0000 max=50842 max_rank=2 min=12561 min_rank=0 imbalance_ratio=1.551575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31585 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25681 max_load_reduction_pct=0.436915 moved_to_replicas=96118 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[35303, 24767, 59355, 24403, 54076, 15485, 19501, 29254] avg=32768.0000 max=59355 max_rank=2 min=15485 min_rank=5 imbalance_ratio=1.811371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] avg=32768.0000 max=33096 max_rank=0 min=31442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14152 max_load_reduction_pct=0.299526 moved_to_replicas=73219 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[35982, 45267, 18285, 37998, 69653, 13599, 24684, 16676] avg=32768.0000 max=69653 max_rank=4 min=13599 min_rank=5 imbalance_ratio=2.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17745 max_load_reduction_pct=0.349022 moved_to_replicas=71494 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[12561, 22511, 50842, 22324, 21257, 41284, 45703, 45662] avg=32768.0000 max=50842 max_rank=2 min=12561 min_rank=0 imbalance_ratio=1.551575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[12561, 22511, 50842, 22324, 21257, 41284, 45703, 45662] avg=32768.0000 max=50842 max_rank=2 min=12561 min_rank=0 imbalance_ratio=1.551575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30942 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14187 max_load_reduction_pct=0.300038 moved_to_replicas=56241 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] avg=32768.0000 max=33098 max_rank=3 min=31461 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=26257 max_load_reduction_pct=0.442372 moved_to_replicas=128555 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] avg=32768.0000 max=33096 max_rank=0 min=31096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36557 max_load_reduction_pct=0.524845 moved_to_replicas=103537 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[39478, 24583, 37640, 47248, 26898, 21200, 41111, 23986] avg=32768.0000 max=47248 max_rank=3 min=21200 min_rank=5 imbalance_ratio=1.441895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17745 max_load_reduction_pct=0.349022 moved_to_replicas=71494 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17745 max_load_reduction_pct=0.349022 moved_to_replicas=71494 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[43778, 33972, 41269, 27420, 22573, 31464, 23303, 38365] avg=32768.0000 max=43778 max_rank=0 min=22573 min_rank=4 imbalance_ratio=1.335999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[8708, 3426, 66610, 15130, 39667, 52261, 20522, 55820] avg=32768.0000 max=66610 max_rank=2 min=3426 min_rank=1 imbalance_ratio=2.032776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[12561, 22511, 50842, 22324, 21257, 41284, 45703, 45662] avg=32768.0000 max=50842 max_rank=2 min=12561 min_rank=0 imbalance_ratio=1.551575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[48617, 15158, 39623, 26023, 30560, 32026, 24898, 45239] avg=32768.0000 max=48617 max_rank=0 min=15158 min_rank=1 imbalance_ratio=1.483673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] avg=32768.0000 max=33096 max_rank=0 min=31442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14152 max_load_reduction_pct=0.299526 moved_to_replicas=73219 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] avg=32768.0000 max=33098 max_rank=0 min=31634 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=10680 max_load_reduction_pct=0.243958 moved_to_replicas=61433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] avg=32768.0000 max=33097 max_rank=3 min=31263 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33513 max_load_reduction_pct=0.503123 moved_to_replicas=159673 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[35303, 24767, 59355, 24403, 54076, 15485, 19501, 29254] avg=32768.0000 max=59355 max_rank=2 min=15485 min_rank=5 imbalance_ratio=1.811371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[8708, 3426, 66610, 15130, 39667, 52261, 20522, 55820] avg=32768.0000 max=66610 max_rank=2 min=3426 min_rank=1 imbalance_ratio=2.032776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17745 max_load_reduction_pct=0.349022 moved_to_replicas=71494 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=15521 max_load_reduction_pct=0.319250 moved_to_replicas=60695 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[8708, 3426, 66610, 15130, 39667, 52261, 20522, 55820] avg=32768.0000 max=66610 max_rank=2 min=3426 min_rank=1 imbalance_ratio=2.032776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[43778, 33972, 41269, 27420, 22573, 31464, 23303, 38365] avg=32768.0000 max=43778 max_rank=0 min=22573 min_rank=4 imbalance_ratio=1.335999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] avg=32768.0000 max=33098 max_rank=3 min=31461 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=26257 max_load_reduction_pct=0.442372 moved_to_replicas=128555 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] avg=32768.0000 max=33097 max_rank=3 min=31263 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33513 max_load_reduction_pct=0.503123 moved_to_replicas=159673 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[12266, 35078, 70503, 54837, 12318, 10254, 49173, 17715] avg=32768.0000 max=70503 max_rank=2 min=10254 min_rank=5 imbalance_ratio=2.151581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] avg=32768.0000 max=33097 max_rank=3 min=31263 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33513 max_load_reduction_pct=0.503123 moved_to_replicas=159673 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[35982, 45267, 18285, 37998, 69653, 13599, 24684, 16676] avg=32768.0000 max=69653 max_rank=4 min=13599 min_rank=5 imbalance_ratio=2.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[8708, 3426, 66610, 15130, 39667, 52261, 20522, 55820] avg=32768.0000 max=66610 max_rank=2 min=3426 min_rank=1 imbalance_ratio=2.032776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[37059, 59975, 34617, 42948, 40846, 25572, 3285, 17842] avg=32768.0000 max=59975 max_rank=1 min=3285 min_rank=6 imbalance_ratio=1.830292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] avg=32768.0000 max=33098 max_rank=0 min=31634 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=10680 max_load_reduction_pct=0.243958 moved_to_replicas=61433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=30471 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37406 max_load_reduction_pct=0.530559 moved_to_replicas=150937 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[48617, 15158, 39623, 26023, 30560, 32026, 24898, 45239] avg=32768.0000 max=48617 max_rank=0 min=15158 min_rank=1 imbalance_ratio=1.483673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] avg=32768.0000 max=33096 max_rank=0 min=31096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36557 max_load_reduction_pct=0.524845 moved_to_replicas=103537 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[35982, 45267, 18285, 37998, 69653, 13599, 24684, 16676] avg=32768.0000 max=69653 max_rank=4 min=13599 min_rank=5 imbalance_ratio=2.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] avg=32768.0000 max=33097 max_rank=3 min=31263 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33513 max_load_reduction_pct=0.503123 moved_to_replicas=159673 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] avg=32768.0000 max=33097 max_rank=3 min=31696 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26878 max_load_reduction_pct=0.448153 moved_to_replicas=88854 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[35982, 45267, 18285, 37998, 69653, 13599, 24684, 16676] avg=32768.0000 max=69653 max_rank=4 min=13599 min_rank=5 imbalance_ratio=2.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=15521 max_load_reduction_pct=0.319250 moved_to_replicas=60695 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[12266, 35078, 70503, 54837, 12318, 10254, 49173, 17715] avg=32768.0000 max=70503 max_rank=2 min=10254 min_rank=5 imbalance_ratio=2.151581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] avg=32768.0000 max=33096 max_rank=0 min=31096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36557 max_load_reduction_pct=0.524845 moved_to_replicas=103537 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[41103, 69582, 53793, 28787, 22039, 14246, 25077, 7517] avg=32768.0000 max=69582 max_rank=1 min=7517 min_rank=7 imbalance_ratio=2.123474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] avg=32768.0000 max=33096 max_rank=0 min=31096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36557 max_load_reduction_pct=0.524845 moved_to_replicas=103537 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[35303, 24767, 59355, 24403, 54076, 15485, 19501, 29254] avg=32768.0000 max=59355 max_rank=2 min=15485 min_rank=5 imbalance_ratio=1.811371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=30471 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37406 max_load_reduction_pct=0.530559 moved_to_replicas=150937 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[49021, 21859, 25132, 20425, 29293, 39739, 42603, 34072] avg=32768.0000 max=49021 max_rank=0 min=20425 min_rank=3 imbalance_ratio=1.496002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[35982, 45267, 18285, 37998, 69653, 13599, 24684, 16676] avg=32768.0000 max=69653 max_rank=4 min=13599 min_rank=5 imbalance_ratio=2.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[37059, 59975, 34617, 42948, 40846, 25572, 3285, 17842] avg=32768.0000 max=59975 max_rank=1 min=3285 min_rank=6 imbalance_ratio=1.830292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] avg=32768.0000 max=33096 max_rank=0 min=31473 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36486 max_load_reduction_pct=0.524360 moved_to_replicas=113366 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[35303, 24767, 59355, 24403, 54076, 15485, 19501, 29254] avg=32768.0000 max=59355 max_rank=2 min=15485 min_rank=5 imbalance_ratio=1.811371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] avg=32768.0000 max=33098 max_rank=3 min=31461 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=26257 max_load_reduction_pct=0.442372 moved_to_replicas=128555 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=15925 max_load_reduction_pct=0.324861 moved_to_replicas=61856 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] avg=32768.0000 max=33097 max_rank=3 min=31696 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26878 max_load_reduction_pct=0.448153 moved_to_replicas=88854 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] avg=32768.0000 max=33096 max_rank=0 min=31096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36557 max_load_reduction_pct=0.524845 moved_to_replicas=103537 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[35303, 24767, 59355, 24403, 54076, 15485, 19501, 29254] avg=32768.0000 max=59355 max_rank=2 min=15485 min_rank=5 imbalance_ratio=1.811371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[41103, 69582, 53793, 28787, 22039, 14246, 25077, 7517] avg=32768.0000 max=69582 max_rank=1 min=7517 min_rank=7 imbalance_ratio=2.123474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] avg=32768.0000 max=33098 max_rank=3 min=31461 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=26257 max_load_reduction_pct=0.442372 moved_to_replicas=128555 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[17432, 34096, 41175, 38016, 25559, 49545, 39563, 16758] avg=32768.0000 max=49545 max_rank=5 min=16758 min_rank=7 imbalance_ratio=1.511993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] avg=32768.0000 max=33098 max_rank=3 min=31461 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=26257 max_load_reduction_pct=0.442372 moved_to_replicas=128555 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[48617, 15158, 39623, 26023, 30560, 32026, 24898, 45239] avg=32768.0000 max=48617 max_rank=0 min=15158 min_rank=1 imbalance_ratio=1.483673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] avg=32768.0000 max=33096 max_rank=0 min=31473 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36486 max_load_reduction_pct=0.524360 moved_to_replicas=113366 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[49021, 21859, 25132, 20425, 29293, 39739, 42603, 34072] avg=32768.0000 max=49021 max_rank=0 min=20425 min_rank=3 imbalance_ratio=1.496002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[35303, 24767, 59355, 24403, 54076, 15485, 19501, 29254] avg=32768.0000 max=59355 max_rank=2 min=15485 min_rank=5 imbalance_ratio=1.811371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31542 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16449 max_load_reduction_pct=0.332001 moved_to_replicas=82564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[48617, 15158, 39623, 26023, 30560, 32026, 24898, 45239] avg=32768.0000 max=48617 max_rank=0 min=15158 min_rank=1 imbalance_ratio=1.483673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=15521 max_load_reduction_pct=0.319250 moved_to_replicas=60695 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=15925 max_load_reduction_pct=0.324861 moved_to_replicas=61856 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] avg=32768.0000 max=33098 max_rank=3 min=31461 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=26257 max_load_reduction_pct=0.442372 moved_to_replicas=128555 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[48617, 15158, 39623, 26023, 30560, 32026, 24898, 45239] avg=32768.0000 max=48617 max_rank=0 min=15158 min_rank=1 imbalance_ratio=1.483673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[17432, 34096, 41175, 38016, 25559, 49545, 39563, 16758] avg=32768.0000 max=49545 max_rank=5 min=16758 min_rank=7 imbalance_ratio=1.511993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=15521 max_load_reduction_pct=0.319250 moved_to_replicas=60695 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[24024, 37006, 47284, 42347, 21548, 15751, 36397, 37787] avg=32768.0000 max=47284 max_rank=2 min=15751 min_rank=5 imbalance_ratio=1.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=15521 max_load_reduction_pct=0.319250 moved_to_replicas=60695 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31542 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16449 max_load_reduction_pct=0.332001 moved_to_replicas=82564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[37059, 59975, 34617, 42948, 40846, 25572, 3285, 17842] avg=32768.0000 max=59975 max_rank=1 min=3285 min_rank=6 imbalance_ratio=1.830292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30942 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14187 max_load_reduction_pct=0.300038 moved_to_replicas=56241 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[48617, 15158, 39623, 26023, 30560, 32026, 24898, 45239] avg=32768.0000 max=48617 max_rank=0 min=15158 min_rank=1 imbalance_ratio=1.483673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[37059, 59975, 34617, 42948, 40846, 25572, 3285, 17842] avg=32768.0000 max=59975 max_rank=1 min=3285 min_rank=6 imbalance_ratio=1.830292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] avg=32768.0000 max=33097 max_rank=3 min=31696 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26878 max_load_reduction_pct=0.448153 moved_to_replicas=88854 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=15521 max_load_reduction_pct=0.319250 moved_to_replicas=60695 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[37059, 59975, 34617, 42948, 40846, 25572, 3285, 17842] avg=32768.0000 max=59975 max_rank=1 min=3285 min_rank=6 imbalance_ratio=1.830292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[24024, 37006, 47284, 42347, 21548, 15751, 36397, 37787] avg=32768.0000 max=47284 max_rank=2 min=15751 min_rank=5 imbalance_ratio=1.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] avg=32768.0000 max=33097 max_rank=3 min=31696 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26878 max_load_reduction_pct=0.448153 moved_to_replicas=88854 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[12561, 22511, 50842, 22324, 21257, 41284, 45703, 45662] avg=32768.0000 max=50842 max_rank=2 min=12561 min_rank=0 imbalance_ratio=1.551575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30942 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14187 max_load_reduction_pct=0.300038 moved_to_replicas=56241 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] avg=32768.0000 max=33097 max_rank=3 min=31696 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26878 max_load_reduction_pct=0.448153 moved_to_replicas=88854 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[49021, 21859, 25132, 20425, 29293, 39739, 42603, 34072] avg=32768.0000 max=49021 max_rank=0 min=20425 min_rank=3 imbalance_ratio=1.496002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17745 max_load_reduction_pct=0.349022 moved_to_replicas=71494 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[37059, 59975, 34617, 42948, 40846, 25572, 3285, 17842] avg=32768.0000 max=59975 max_rank=1 min=3285 min_rank=6 imbalance_ratio=1.830292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[49021, 21859, 25132, 20425, 29293, 39739, 42603, 34072] avg=32768.0000 max=49021 max_rank=0 min=20425 min_rank=3 imbalance_ratio=1.496002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=15925 max_load_reduction_pct=0.324861 moved_to_replicas=61856 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] avg=32768.0000 max=33097 max_rank=3 min=31696 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26878 max_load_reduction_pct=0.448153 moved_to_replicas=88854 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[12561, 22511, 50842, 22324, 21257, 41284, 45703, 45662] avg=32768.0000 max=50842 max_rank=2 min=12561 min_rank=0 imbalance_ratio=1.551575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[49021, 21859, 25132, 20425, 29293, 39739, 42603, 34072] avg=32768.0000 max=49021 max_rank=0 min=20425 min_rank=3 imbalance_ratio=1.496002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=15925 max_load_reduction_pct=0.324861 moved_to_replicas=61856 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[8708, 3426, 66610, 15130, 39667, 52261, 20522, 55820] avg=32768.0000 max=66610 max_rank=2 min=3426 min_rank=1 imbalance_ratio=2.032776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=1048800 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=55296 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17745 max_load_reduction_pct=0.349022 moved_to_replicas=71494 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=15925 max_load_reduction_pct=0.324861 moved_to_replicas=61856 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[49021, 21859, 25132, 20425, 29293, 39739, 42603, 34072] avg=32768.0000 max=49021 max_rank=0 min=20425 min_rank=3 imbalance_ratio=1.496002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] avg=32768.0000 max=33097 max_rank=3 min=31263 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33513 max_load_reduction_pct=0.503123 moved_to_replicas=159673 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=15925 max_load_reduction_pct=0.324861 moved_to_replicas=61856 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[8708, 3426, 66610, 15130, 39667, 52261, 20522, 55820] avg=32768.0000 max=66610 max_rank=2 min=3426 min_rank=1 imbalance_ratio=2.032776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[35982, 45267, 18285, 37998, 69653, 13599, 24684, 16676] avg=32768.0000 max=69653 max_rank=4 min=13599 min_rank=5 imbalance_ratio=2.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=32 total_expected_recv=1051754 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=52224 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] avg=32768.0000 max=33097 max_rank=3 min=31263 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33513 max_load_reduction_pct=0.503123 moved_to_replicas=159673 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] avg=32768.0000 max=33096 max_rank=0 min=31096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36557 max_load_reduction_pct=0.524845 moved_to_replicas=103537 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[35982, 45267, 18285, 37998, 69653, 13599, 24684, 16676] avg=32768.0000 max=69653 max_rank=4 min=13599 min_rank=5 imbalance_ratio=2.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[35303, 24767, 59355, 24403, 54076, 15485, 19501, 29254] avg=32768.0000 max=59355 max_rank=2 min=15485 min_rank=5 imbalance_ratio=1.811371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] avg=32768.0000 max=33096 max_rank=0 min=31096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36557 max_load_reduction_pct=0.524845 moved_to_replicas=103537 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] avg=32768.0000 max=33098 max_rank=3 min=31461 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=26257 max_load_reduction_pct=0.442372 moved_to_replicas=128555 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[35303, 24767, 59355, 24403, 54076, 15485, 19501, 29254] avg=32768.0000 max=59355 max_rank=2 min=15485 min_rank=5 imbalance_ratio=1.811371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[48617, 15158, 39623, 26023, 30560, 32026, 24898, 45239] avg=32768.0000 max=48617 max_rank=0 min=15158 min_rank=1 imbalance_ratio=1.483673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] avg=32768.0000 max=33098 max_rank=3 min=31461 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=26257 max_load_reduction_pct=0.442372 moved_to_replicas=128555 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=32 total_expected_recv=1052426 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=52608 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=15521 max_load_reduction_pct=0.319250 moved_to_replicas=60695 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=32 total_expected_recv=1048459 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=55168 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[48617, 15158, 39623, 26023, 30560, 32026, 24898, 45239] avg=32768.0000 max=48617 max_rank=0 min=15158 min_rank=1 imbalance_ratio=1.483673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[37059, 59975, 34617, 42948, 40846, 25572, 3285, 17842] avg=32768.0000 max=59975 max_rank=1 min=3285 min_rank=6 imbalance_ratio=1.830292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=15521 max_load_reduction_pct=0.319250 moved_to_replicas=60695 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=32 total_expected_recv=1046514 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=58496 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] avg=32768.0000 max=33097 max_rank=3 min=31696 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26878 max_load_reduction_pct=0.448153 moved_to_replicas=88854 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=32 total_expected_recv=1045859 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=58752 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[37059, 59975, 34617, 42948, 40846, 25572, 3285, 17842] avg=32768.0000 max=59975 max_rank=1 min=3285 min_rank=6 imbalance_ratio=1.830292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[49021, 21859, 25132, 20425, 29293, 39739, 42603, 34072] avg=32768.0000 max=49021 max_rank=0 min=20425 min_rank=3 imbalance_ratio=1.496002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] avg=32768.0000 max=33097 max_rank=3 min=31696 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26878 max_load_reduction_pct=0.448153 moved_to_replicas=88854 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=15925 max_load_reduction_pct=0.324861 moved_to_replicas=61856 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[49021, 21859, 25132, 20425, 29293, 39739, 42603, 34072] avg=32768.0000 max=49021 max_rank=0 min=20425 min_rank=3 imbalance_ratio=1.496002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=15925 max_load_reduction_pct=0.324861 moved_to_replicas=61856 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=32 total_expected_recv=1052841 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=50816 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=32 total_expected_recv=1041955 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=61184 peak_demand=33097 required_capacity=1.010040 +Number of parameters in transformer block in billions: 23.22 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 + [2026-08-30 05:11:40] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 27931.9 | throughput per GPU (TFLOP/s/GPU): 11.9 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102622E+01 | loss scale: 1.0 | grad norm: 44.467 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33536 | min safe capacity: 1.0234 |Theoretical memory footprints: weight and optimizer=169029.43 MB + +[Rank 0] (after 1 iterations) memory (MB) | allocated: 58091.849609375 | max allocated: 74283.916015625 | reserved: 75842.0 | max reserved: 75842.0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=34816 expected_recv=66054 expected_overflow=0 exact_recv_per_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=34816 expected_recv=65318 expected_overflow=0 exact_recv_per_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=34816 expected_recv=64303 expected_overflow=0 exact_recv_per_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=34816 expected_recv=63846 expected_overflow=0 exact_recv_per_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=34816 expected_recv=66054 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=34816 expected_recv=65318 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=34816 expected_recv=64303 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=34816 expected_recv=63846 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=64896 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=64256 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=34816 expected_recv=64513 expected_overflow=0 exact_recv_per_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=34816 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=34816 expected_recv=65405 expected_overflow=0 exact_recv_per_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=34816 expected_recv=64893 expected_overflow=0 exact_recv_per_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=34816 expected_recv=64513 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=34816 expected_recv=66194 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=34816 expected_recv=64708 expected_overflow=0 exact_recv_per_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=34816 expected_recv=65405 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=34816 expected_recv=64893 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=34816 expected_recv=64708 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=34816 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=34816 expected_recv=63872 expected_overflow=0 exact_recv_per_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=34816 expected_recv=66194 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=34816 expected_recv=65594 expected_overflow=0 exact_recv_per_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=34816 expected_recv=64554 expected_overflow=0 exact_recv_per_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=34816 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=34816 expected_recv=63872 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=34816 expected_recv=65498 expected_overflow=0 exact_recv_per_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=34816 expected_recv=65594 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=34816 expected_recv=64554 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=34816 expected_recv=66190 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=64512 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=34816 expected_recv=65498 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=34816 expected_recv=63037 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=34816 expected_recv=65951 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=34816 expected_recv=64882 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=34816 expected_recv=63037 actual_valid_recv=63616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=34816 expected_recv=65951 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=34816 expected_recv=64882 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=34816 expected_recv=65650 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=63616 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=34816 expected_recv=65650 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=34816 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=34816 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=34816 expected_recv=63568 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=64512 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=34816 expected_recv=63568 actual_valid_recv=64000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=34816 expected_recv=64640 expected_overflow=0 exact_recv_per_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=64000 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=34816 expected_recv=64640 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=34816 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=34816 expected_recv=63568 actual_valid_recv=64128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=34816 expected_recv=65119 expected_overflow=0 exact_recv_per_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=64128 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=34816 expected_recv=62098 expected_overflow=0 exact_recv_per_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=34816 expected_recv=65119 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=34816 expected_recv=62098 actual_valid_recv=62848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=62848 slack_slots=6784 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=34816 expected_recv=65317 expected_overflow=0 exact_recv_per_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=34816 expected_recv=65317 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=34816 expected_recv=66017 expected_overflow=0 exact_recv_per_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=34816 expected_recv=66086 expected_overflow=0 exact_recv_per_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=34816 expected_recv=66017 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=34816 expected_recv=66086 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=34816 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=34816 expected_recv=66194 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=34816 expected_recv=63566 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=34816 expected_recv=65504 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=34816 expected_recv=64256 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=34816 expected_recv=63566 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=34816 expected_recv=65504 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=64256 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=34816 expected_recv=64256 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65024 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=34816 expected_recv=64138 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=34816 expected_recv=64138 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65024 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=34816 expected_recv=64605 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=34816 expected_recv=64585 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=34816 expected_recv=64605 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=34816 expected_recv=64585 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=34816 expected_recv=65235 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=34816 expected_recv=65235 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=34816 expected_recv=61899 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=34816 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=34816 expected_recv=61899 actual_valid_recv=62976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=34816 expected_recv=66194 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=62976 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=34816 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=34816 expected_recv=63568 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=64384 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=34816 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=34816 expected_recv=63568 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=64384 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=34816 expected_recv=63704 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=34816 expected_recv=63704 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=64384 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=34816 expected_recv=64852 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=34816 expected_recv=64773 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=34816 expected_recv=64852 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=34816 expected_recv=64773 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=34816 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=34816 expected_recv=65527 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=34816 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=34816 expected_recv=64942 expected_overflow=0 exact_recv_per_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=34816 expected_recv=63568 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=34816 expected_recv=64232 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=34816 expected_recv=64942 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=64256 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=34816 expected_recv=65128 expected_overflow=0 exact_recv_per_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=34816 expected_recv=64232 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=34816 expected_recv=65128 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65024 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=34816 expected_recv=64743 expected_overflow=0 exact_recv_per_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=34816 expected_recv=65635 expected_overflow=0 exact_recv_per_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=34816 expected_recv=64743 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=34816 expected_recv=65635 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=34816 expected_recv=63849 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=34816 expected_recv=63849 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=34816 expected_recv=63766 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=34816 expected_recv=65264 expected_overflow=0 exact_recv_per_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=34816 expected_recv=65822 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=64512 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=34816 expected_recv=63766 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=34816 expected_recv=65264 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=34816 expected_recv=65822 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=64512 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=34816 expected_recv=66084 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=34816 expected_recv=66084 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=34816 expected_recv=61952 expected_overflow=0 exact_recv_per_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=34816 expected_recv=61952 actual_valid_recv=62720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[32819, 30237, 26421, 20996, 22180, 51615, 30887, 46989] avg=32768.0000 max=51615 max_rank=5 min=20996 min_rank=3 imbalance_ratio=1.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=62720 slack_slots=6912 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=34816 expected_recv=65185 expected_overflow=0 exact_recv_per_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[32819, 30237, 26421, 20996, 22180, 51615, 30887, 46989] avg=32768.0000 max=51615 max_rank=5 min=20996 min_rank=3 imbalance_ratio=1.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[32819, 30237, 26421, 20996, 22180, 51615, 30887, 46989] avg=32768.0000 max=51615 max_rank=5 min=20996 min_rank=3 imbalance_ratio=1.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=34816 expected_recv=65185 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[32819, 30237, 26421, 20996, 22180, 51615, 30887, 46989] avg=32768.0000 max=51615 max_rank=5 min=20996 min_rank=3 imbalance_ratio=1.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] avg=32768.0000 max=33097 max_rank=5 min=30887 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18518 max_load_reduction_pct=0.358772 moved_to_replicas=61334 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] avg=32768.0000 max=33097 max_rank=5 min=30887 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18518 max_load_reduction_pct=0.358772 moved_to_replicas=61334 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[32819, 30237, 26421, 20996, 22180, 51615, 30887, 46989] avg=32768.0000 max=51615 max_rank=5 min=20996 min_rank=3 imbalance_ratio=1.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] avg=32768.0000 max=33097 max_rank=5 min=30887 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18518 max_load_reduction_pct=0.358772 moved_to_replicas=61334 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[32819, 30237, 26421, 20996, 22180, 51615, 30887, 46989] avg=32768.0000 max=51615 max_rank=5 min=20996 min_rank=3 imbalance_ratio=1.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] avg=32768.0000 max=33097 max_rank=5 min=30887 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18518 max_load_reduction_pct=0.358772 moved_to_replicas=61334 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[32819, 30237, 26421, 20996, 22180, 51615, 30887, 46989] avg=32768.0000 max=51615 max_rank=5 min=20996 min_rank=3 imbalance_ratio=1.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] avg=32768.0000 max=33097 max_rank=5 min=30887 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18518 max_load_reduction_pct=0.358772 moved_to_replicas=61334 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[19440, 68443, 18830, 38447, 36834, 2641, 44572, 32937] avg=32768.0000 max=68443 max_rank=1 min=2641 min_rank=5 imbalance_ratio=2.088715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[32819, 30237, 26421, 20996, 22180, 51615, 30887, 46989] avg=32768.0000 max=51615 max_rank=5 min=20996 min_rank=3 imbalance_ratio=1.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[19440, 68443, 18830, 38447, 36834, 2641, 44572, 32937] avg=32768.0000 max=68443 max_rank=1 min=2641 min_rank=5 imbalance_ratio=2.088715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] avg=32768.0000 max=33097 max_rank=5 min=30887 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18518 max_load_reduction_pct=0.358772 moved_to_replicas=61334 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] avg=32768.0000 max=33097 max_rank=5 min=30887 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18518 max_load_reduction_pct=0.358772 moved_to_replicas=61334 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[19440, 68443, 18830, 38447, 36834, 2641, 44572, 32937] avg=32768.0000 max=68443 max_rank=1 min=2641 min_rank=5 imbalance_ratio=2.088715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[19440, 68443, 18830, 38447, 36834, 2641, 44572, 32937] avg=32768.0000 max=68443 max_rank=1 min=2641 min_rank=5 imbalance_ratio=2.088715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] avg=32768.0000 max=33097 max_rank=2 min=31418 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35346 max_load_reduction_pct=0.516430 moved_to_replicas=118233 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] avg=32768.0000 max=33097 max_rank=5 min=30887 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18518 max_load_reduction_pct=0.358772 moved_to_replicas=61334 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] avg=32768.0000 max=33097 max_rank=2 min=31418 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35346 max_load_reduction_pct=0.516430 moved_to_replicas=118233 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[19440, 68443, 18830, 38447, 36834, 2641, 44572, 32937] avg=32768.0000 max=68443 max_rank=1 min=2641 min_rank=5 imbalance_ratio=2.088715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] avg=32768.0000 max=33097 max_rank=2 min=31418 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35346 max_load_reduction_pct=0.516430 moved_to_replicas=118233 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[19440, 68443, 18830, 38447, 36834, 2641, 44572, 32937] avg=32768.0000 max=68443 max_rank=1 min=2641 min_rank=5 imbalance_ratio=2.088715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] avg=32768.0000 max=33097 max_rank=2 min=31418 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35346 max_load_reduction_pct=0.516430 moved_to_replicas=118233 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[19440, 68443, 18830, 38447, 36834, 2641, 44572, 32937] avg=32768.0000 max=68443 max_rank=1 min=2641 min_rank=5 imbalance_ratio=2.088715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] avg=32768.0000 max=33097 max_rank=2 min=31418 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35346 max_load_reduction_pct=0.516430 moved_to_replicas=118233 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[67697, 32402, 38323, 24276, 54722, 15725, 9034, 19965] avg=32768.0000 max=67697 max_rank=0 min=9034 min_rank=6 imbalance_ratio=2.065948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[67697, 32402, 38323, 24276, 54722, 15725, 9034, 19965] avg=32768.0000 max=67697 max_rank=0 min=9034 min_rank=6 imbalance_ratio=2.065948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] avg=32768.0000 max=33097 max_rank=2 min=31418 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35346 max_load_reduction_pct=0.516430 moved_to_replicas=118233 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[19440, 68443, 18830, 38447, 36834, 2641, 44572, 32937] avg=32768.0000 max=68443 max_rank=1 min=2641 min_rank=5 imbalance_ratio=2.088715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] avg=32768.0000 max=33097 max_rank=2 min=31418 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35346 max_load_reduction_pct=0.516430 moved_to_replicas=118233 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[67697, 32402, 38323, 24276, 54722, 15725, 9034, 19965] avg=32768.0000 max=67697 max_rank=0 min=9034 min_rank=6 imbalance_ratio=2.065948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[67697, 32402, 38323, 24276, 54722, 15725, 9034, 19965] avg=32768.0000 max=67697 max_rank=0 min=9034 min_rank=6 imbalance_ratio=2.065948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=31490 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34600 max_load_reduction_pct=0.511101 moved_to_replicas=95117 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=31490 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34600 max_load_reduction_pct=0.511101 moved_to_replicas=95117 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] avg=32768.0000 max=33097 max_rank=2 min=31418 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35346 max_load_reduction_pct=0.516430 moved_to_replicas=118233 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[67697, 32402, 38323, 24276, 54722, 15725, 9034, 19965] avg=32768.0000 max=67697 max_rank=0 min=9034 min_rank=6 imbalance_ratio=2.065948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[67697, 32402, 38323, 24276, 54722, 15725, 9034, 19965] avg=32768.0000 max=67697 max_rank=0 min=9034 min_rank=6 imbalance_ratio=2.065948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=31490 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34600 max_load_reduction_pct=0.511101 moved_to_replicas=95117 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=31490 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34600 max_load_reduction_pct=0.511101 moved_to_replicas=95117 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[67697, 32402, 38323, 24276, 54722, 15725, 9034, 19965] avg=32768.0000 max=67697 max_rank=0 min=9034 min_rank=6 imbalance_ratio=2.065948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=31490 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34600 max_load_reduction_pct=0.511101 moved_to_replicas=95117 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[43762, 23555, 26659, 42638, 36575, 21358, 35811, 31786] avg=32768.0000 max=43762 max_rank=0 min=21358 min_rank=5 imbalance_ratio=1.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=31490 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34600 max_load_reduction_pct=0.511101 moved_to_replicas=95117 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[43762, 23555, 26659, 42638, 36575, 21358, 35811, 31786] avg=32768.0000 max=43762 max_rank=0 min=21358 min_rank=5 imbalance_ratio=1.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[67697, 32402, 38323, 24276, 54722, 15725, 9034, 19965] avg=32768.0000 max=67697 max_rank=0 min=9034 min_rank=6 imbalance_ratio=2.065948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[43762, 23555, 26659, 42638, 36575, 21358, 35811, 31786] avg=32768.0000 max=43762 max_rank=0 min=21358 min_rank=5 imbalance_ratio=1.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=31490 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34600 max_load_reduction_pct=0.511101 moved_to_replicas=95117 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] avg=32768.0000 max=33097 max_rank=0 min=31786 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=10665 max_load_reduction_pct=0.243705 moved_to_replicas=46681 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[43762, 23555, 26659, 42638, 36575, 21358, 35811, 31786] avg=32768.0000 max=43762 max_rank=0 min=21358 min_rank=5 imbalance_ratio=1.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[43762, 23555, 26659, 42638, 36575, 21358, 35811, 31786] avg=32768.0000 max=43762 max_rank=0 min=21358 min_rank=5 imbalance_ratio=1.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] avg=32768.0000 max=33097 max_rank=0 min=31786 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=10665 max_load_reduction_pct=0.243705 moved_to_replicas=46681 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=31490 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34600 max_load_reduction_pct=0.511101 moved_to_replicas=95117 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] avg=32768.0000 max=33097 max_rank=0 min=31786 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=10665 max_load_reduction_pct=0.243705 moved_to_replicas=46681 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[43762, 23555, 26659, 42638, 36575, 21358, 35811, 31786] avg=32768.0000 max=43762 max_rank=0 min=21358 min_rank=5 imbalance_ratio=1.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] avg=32768.0000 max=33097 max_rank=0 min=31786 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=10665 max_load_reduction_pct=0.243705 moved_to_replicas=46681 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[51993, 40899, 55140, 20227, 11450, 20345, 14148, 47942] avg=32768.0000 max=55140 max_rank=2 min=11450 min_rank=4 imbalance_ratio=1.682739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] avg=32768.0000 max=33097 max_rank=0 min=31786 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=10665 max_load_reduction_pct=0.243705 moved_to_replicas=46681 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[43762, 23555, 26659, 42638, 36575, 21358, 35811, 31786] avg=32768.0000 max=43762 max_rank=0 min=21358 min_rank=5 imbalance_ratio=1.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] avg=32768.0000 max=33097 max_rank=0 min=31786 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=10665 max_load_reduction_pct=0.243705 moved_to_replicas=46681 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[43762, 23555, 26659, 42638, 36575, 21358, 35811, 31786] avg=32768.0000 max=43762 max_rank=0 min=21358 min_rank=5 imbalance_ratio=1.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[51993, 40899, 55140, 20227, 11450, 20345, 14148, 47942] avg=32768.0000 max=55140 max_rank=2 min=11450 min_rank=4 imbalance_ratio=1.682739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[51993, 40899, 55140, 20227, 11450, 20345, 14148, 47942] avg=32768.0000 max=55140 max_rank=2 min=11450 min_rank=4 imbalance_ratio=1.682739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=22043 max_load_reduction_pct=0.399764 moved_to_replicas=112959 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[51993, 40899, 55140, 20227, 11450, 20345, 14148, 47942] avg=32768.0000 max=55140 max_rank=2 min=11450 min_rank=4 imbalance_ratio=1.682739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[51993, 40899, 55140, 20227, 11450, 20345, 14148, 47942] avg=32768.0000 max=55140 max_rank=2 min=11450 min_rank=4 imbalance_ratio=1.682739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] avg=32768.0000 max=33097 max_rank=0 min=31786 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=10665 max_load_reduction_pct=0.243705 moved_to_replicas=46681 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[51993, 40899, 55140, 20227, 11450, 20345, 14148, 47942] avg=32768.0000 max=55140 max_rank=2 min=11450 min_rank=4 imbalance_ratio=1.682739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=22043 max_load_reduction_pct=0.399764 moved_to_replicas=112959 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=22043 max_load_reduction_pct=0.399764 moved_to_replicas=112959 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] avg=32768.0000 max=33097 max_rank=0 min=31786 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=10665 max_load_reduction_pct=0.243705 moved_to_replicas=46681 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[18184, 32023, 50256, 65266, 13240, 7834, 55043, 20298] avg=32768.0000 max=65266 max_rank=3 min=7834 min_rank=5 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=22043 max_load_reduction_pct=0.399764 moved_to_replicas=112959 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=22043 max_load_reduction_pct=0.399764 moved_to_replicas=112959 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=22043 max_load_reduction_pct=0.399764 moved_to_replicas=112959 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31544 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32169 max_load_reduction_pct=0.492891 moved_to_replicas=134809 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[51993, 40899, 55140, 20227, 11450, 20345, 14148, 47942] avg=32768.0000 max=55140 max_rank=2 min=11450 min_rank=4 imbalance_ratio=1.682739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[18184, 32023, 50256, 65266, 13240, 7834, 55043, 20298] avg=32768.0000 max=65266 max_rank=3 min=7834 min_rank=5 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[18184, 32023, 50256, 65266, 13240, 7834, 55043, 20298] avg=32768.0000 max=65266 max_rank=3 min=7834 min_rank=5 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[51993, 40899, 55140, 20227, 11450, 20345, 14148, 47942] avg=32768.0000 max=55140 max_rank=2 min=11450 min_rank=4 imbalance_ratio=1.682739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[18184, 32023, 50256, 65266, 13240, 7834, 55043, 20298] avg=32768.0000 max=65266 max_rank=3 min=7834 min_rank=5 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[18184, 32023, 50256, 65266, 13240, 7834, 55043, 20298] avg=32768.0000 max=65266 max_rank=3 min=7834 min_rank=5 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[18184, 32023, 50256, 65266, 13240, 7834, 55043, 20298] avg=32768.0000 max=65266 max_rank=3 min=7834 min_rank=5 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=22043 max_load_reduction_pct=0.399764 moved_to_replicas=112959 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31544 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32169 max_load_reduction_pct=0.492891 moved_to_replicas=134809 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31544 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32169 max_load_reduction_pct=0.492891 moved_to_replicas=134809 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[30358, 29747, 71433, 19440, 48778, 10974, 33360, 18054] avg=32768.0000 max=71433 max_rank=2 min=10974 min_rank=5 imbalance_ratio=2.179962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=22043 max_load_reduction_pct=0.399764 moved_to_replicas=112959 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31544 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32169 max_load_reduction_pct=0.492891 moved_to_replicas=134809 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31544 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32169 max_load_reduction_pct=0.492891 moved_to_replicas=134809 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31544 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32169 max_load_reduction_pct=0.492891 moved_to_replicas=134809 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38336 max_load_reduction_pct=0.536671 moved_to_replicas=91766 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[18184, 32023, 50256, 65266, 13240, 7834, 55043, 20298] avg=32768.0000 max=65266 max_rank=3 min=7834 min_rank=5 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[30358, 29747, 71433, 19440, 48778, 10974, 33360, 18054] avg=32768.0000 max=71433 max_rank=2 min=10974 min_rank=5 imbalance_ratio=2.179962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[30358, 29747, 71433, 19440, 48778, 10974, 33360, 18054] avg=32768.0000 max=71433 max_rank=2 min=10974 min_rank=5 imbalance_ratio=2.179962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[18184, 32023, 50256, 65266, 13240, 7834, 55043, 20298] avg=32768.0000 max=65266 max_rank=3 min=7834 min_rank=5 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[30358, 29747, 71433, 19440, 48778, 10974, 33360, 18054] avg=32768.0000 max=71433 max_rank=2 min=10974 min_rank=5 imbalance_ratio=2.179962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[30358, 29747, 71433, 19440, 48778, 10974, 33360, 18054] avg=32768.0000 max=71433 max_rank=2 min=10974 min_rank=5 imbalance_ratio=2.179962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[30358, 29747, 71433, 19440, 48778, 10974, 33360, 18054] avg=32768.0000 max=71433 max_rank=2 min=10974 min_rank=5 imbalance_ratio=2.179962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31544 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32169 max_load_reduction_pct=0.492891 moved_to_replicas=134809 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38336 max_load_reduction_pct=0.536671 moved_to_replicas=91766 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[18203, 46751, 40589, 40296, 10305, 43271, 40091, 22638] avg=32768.0000 max=46751 max_rank=1 min=10305 min_rank=4 imbalance_ratio=1.426727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38336 max_load_reduction_pct=0.536671 moved_to_replicas=91766 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31544 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32169 max_load_reduction_pct=0.492891 moved_to_replicas=134809 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38336 max_load_reduction_pct=0.536671 moved_to_replicas=91766 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38336 max_load_reduction_pct=0.536671 moved_to_replicas=91766 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38336 max_load_reduction_pct=0.536671 moved_to_replicas=91766 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] avg=32768.0000 max=33097 max_rank=0 min=31160 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=13654 max_load_reduction_pct=0.292058 moved_to_replicas=106029 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[30358, 29747, 71433, 19440, 48778, 10974, 33360, 18054] avg=32768.0000 max=71433 max_rank=2 min=10974 min_rank=5 imbalance_ratio=2.179962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[18203, 46751, 40589, 40296, 10305, 43271, 40091, 22638] avg=32768.0000 max=46751 max_rank=1 min=10305 min_rank=4 imbalance_ratio=1.426727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[18203, 46751, 40589, 40296, 10305, 43271, 40091, 22638] avg=32768.0000 max=46751 max_rank=1 min=10305 min_rank=4 imbalance_ratio=1.426727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[30358, 29747, 71433, 19440, 48778, 10974, 33360, 18054] avg=32768.0000 max=71433 max_rank=2 min=10974 min_rank=5 imbalance_ratio=2.179962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[18203, 46751, 40589, 40296, 10305, 43271, 40091, 22638] avg=32768.0000 max=46751 max_rank=1 min=10305 min_rank=4 imbalance_ratio=1.426727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[18203, 46751, 40589, 40296, 10305, 43271, 40091, 22638] avg=32768.0000 max=46751 max_rank=1 min=10305 min_rank=4 imbalance_ratio=1.426727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38336 max_load_reduction_pct=0.536671 moved_to_replicas=91766 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] avg=32768.0000 max=33097 max_rank=0 min=31160 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=13654 max_load_reduction_pct=0.292058 moved_to_replicas=106029 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[17522, 36901, 52335, 45105, 20517, 20282, 36223, 33259] avg=32768.0000 max=52335 max_rank=2 min=17522 min_rank=0 imbalance_ratio=1.597137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] avg=32768.0000 max=33097 max_rank=0 min=31160 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=13654 max_load_reduction_pct=0.292058 moved_to_replicas=106029 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38336 max_load_reduction_pct=0.536671 moved_to_replicas=91766 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[18203, 46751, 40589, 40296, 10305, 43271, 40091, 22638] avg=32768.0000 max=46751 max_rank=1 min=10305 min_rank=4 imbalance_ratio=1.426727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] avg=32768.0000 max=33097 max_rank=0 min=31160 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=13654 max_load_reduction_pct=0.292058 moved_to_replicas=106029 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] avg=32768.0000 max=33097 max_rank=0 min=31160 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=13654 max_load_reduction_pct=0.292058 moved_to_replicas=106029 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31043 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=19238 max_load_reduction_pct=0.367593 moved_to_replicas=65890 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[18203, 46751, 40589, 40296, 10305, 43271, 40091, 22638] avg=32768.0000 max=46751 max_rank=1 min=10305 min_rank=4 imbalance_ratio=1.426727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[17522, 36901, 52335, 45105, 20517, 20282, 36223, 33259] avg=32768.0000 max=52335 max_rank=2 min=17522 min_rank=0 imbalance_ratio=1.597137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[17522, 36901, 52335, 45105, 20517, 20282, 36223, 33259] avg=32768.0000 max=52335 max_rank=2 min=17522 min_rank=0 imbalance_ratio=1.597137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] avg=32768.0000 max=33097 max_rank=0 min=31160 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=13654 max_load_reduction_pct=0.292058 moved_to_replicas=106029 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[18203, 46751, 40589, 40296, 10305, 43271, 40091, 22638] avg=32768.0000 max=46751 max_rank=1 min=10305 min_rank=4 imbalance_ratio=1.426727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[17522, 36901, 52335, 45105, 20517, 20282, 36223, 33259] avg=32768.0000 max=52335 max_rank=2 min=17522 min_rank=0 imbalance_ratio=1.597137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[17522, 36901, 52335, 45105, 20517, 20282, 36223, 33259] avg=32768.0000 max=52335 max_rank=2 min=17522 min_rank=0 imbalance_ratio=1.597137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] avg=32768.0000 max=33097 max_rank=0 min=31160 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=13654 max_load_reduction_pct=0.292058 moved_to_replicas=106029 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31043 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=19238 max_load_reduction_pct=0.367593 moved_to_replicas=65890 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[17141, 38895, 36571, 33772, 24822, 38998, 26341, 45604] avg=32768.0000 max=45604 max_rank=7 min=17141 min_rank=0 imbalance_ratio=1.391724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31043 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=19238 max_load_reduction_pct=0.367593 moved_to_replicas=65890 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] avg=32768.0000 max=33097 max_rank=0 min=31160 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=13654 max_load_reduction_pct=0.292058 moved_to_replicas=106029 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31043 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=19238 max_load_reduction_pct=0.367593 moved_to_replicas=65890 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[17522, 36901, 52335, 45105, 20517, 20282, 36223, 33259] avg=32768.0000 max=52335 max_rank=2 min=17522 min_rank=0 imbalance_ratio=1.597137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31043 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=19238 max_load_reduction_pct=0.367593 moved_to_replicas=65890 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] avg=32768.0000 max=33097 max_rank=1 min=31428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12507 max_load_reduction_pct=0.274252 moved_to_replicas=64817 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[17522, 36901, 52335, 45105, 20517, 20282, 36223, 33259] avg=32768.0000 max=52335 max_rank=2 min=17522 min_rank=0 imbalance_ratio=1.597137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[17141, 38895, 36571, 33772, 24822, 38998, 26341, 45604] avg=32768.0000 max=45604 max_rank=7 min=17141 min_rank=0 imbalance_ratio=1.391724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[17141, 38895, 36571, 33772, 24822, 38998, 26341, 45604] avg=32768.0000 max=45604 max_rank=7 min=17141 min_rank=0 imbalance_ratio=1.391724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31043 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=19238 max_load_reduction_pct=0.367593 moved_to_replicas=65890 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[17522, 36901, 52335, 45105, 20517, 20282, 36223, 33259] avg=32768.0000 max=52335 max_rank=2 min=17522 min_rank=0 imbalance_ratio=1.597137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[17141, 38895, 36571, 33772, 24822, 38998, 26341, 45604] avg=32768.0000 max=45604 max_rank=7 min=17141 min_rank=0 imbalance_ratio=1.391724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[17141, 38895, 36571, 33772, 24822, 38998, 26341, 45604] avg=32768.0000 max=45604 max_rank=7 min=17141 min_rank=0 imbalance_ratio=1.391724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[15517, 6296, 56463, 37805, 58988, 49310, 3281, 34484] avg=32768.0000 max=58988 max_rank=4 min=3281 min_rank=6 imbalance_ratio=1.800171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31043 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=19238 max_load_reduction_pct=0.367593 moved_to_replicas=65890 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] avg=32768.0000 max=33097 max_rank=1 min=31428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12507 max_load_reduction_pct=0.274252 moved_to_replicas=64817 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] avg=32768.0000 max=33097 max_rank=1 min=31428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12507 max_load_reduction_pct=0.274252 moved_to_replicas=64817 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31043 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=19238 max_load_reduction_pct=0.367593 moved_to_replicas=65890 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] avg=32768.0000 max=33097 max_rank=1 min=31428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12507 max_load_reduction_pct=0.274252 moved_to_replicas=64817 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[17141, 38895, 36571, 33772, 24822, 38998, 26341, 45604] avg=32768.0000 max=45604 max_rank=7 min=17141 min_rank=0 imbalance_ratio=1.391724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] avg=32768.0000 max=33097 max_rank=1 min=31428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12507 max_load_reduction_pct=0.274252 moved_to_replicas=64817 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] avg=32768.0000 max=33097 max_rank=7 min=30471 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25891 max_load_reduction_pct=0.438920 moved_to_replicas=147815 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[17141, 38895, 36571, 33772, 24822, 38998, 26341, 45604] avg=32768.0000 max=45604 max_rank=7 min=17141 min_rank=0 imbalance_ratio=1.391724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[15517, 6296, 56463, 37805, 58988, 49310, 3281, 34484] avg=32768.0000 max=58988 max_rank=4 min=3281 min_rank=6 imbalance_ratio=1.800171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] avg=32768.0000 max=33097 max_rank=1 min=31428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12507 max_load_reduction_pct=0.274252 moved_to_replicas=64817 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[15517, 6296, 56463, 37805, 58988, 49310, 3281, 34484] avg=32768.0000 max=58988 max_rank=4 min=3281 min_rank=6 imbalance_ratio=1.800171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[15517, 6296, 56463, 37805, 58988, 49310, 3281, 34484] avg=32768.0000 max=58988 max_rank=4 min=3281 min_rank=6 imbalance_ratio=1.800171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[17141, 38895, 36571, 33772, 24822, 38998, 26341, 45604] avg=32768.0000 max=45604 max_rank=7 min=17141 min_rank=0 imbalance_ratio=1.391724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] avg=32768.0000 max=33097 max_rank=1 min=31428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12507 max_load_reduction_pct=0.274252 moved_to_replicas=64817 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[15517, 6296, 56463, 37805, 58988, 49310, 3281, 34484] avg=32768.0000 max=58988 max_rank=4 min=3281 min_rank=6 imbalance_ratio=1.800171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] avg=32768.0000 max=33097 max_rank=7 min=30471 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25891 max_load_reduction_pct=0.438920 moved_to_replicas=147815 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[50913, 44241, 30608, 29223, 47040, 11147, 22660, 26312] avg=32768.0000 max=50913 max_rank=0 min=11147 min_rank=5 imbalance_ratio=1.553741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] avg=32768.0000 max=33097 max_rank=7 min=30471 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25891 max_load_reduction_pct=0.438920 moved_to_replicas=147815 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] avg=32768.0000 max=33097 max_rank=7 min=30471 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25891 max_load_reduction_pct=0.438920 moved_to_replicas=147815 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] avg=32768.0000 max=33097 max_rank=1 min=31428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12507 max_load_reduction_pct=0.274252 moved_to_replicas=64817 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[15517, 6296, 56463, 37805, 58988, 49310, 3281, 34484] avg=32768.0000 max=58988 max_rank=4 min=3281 min_rank=6 imbalance_ratio=1.800171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] avg=32768.0000 max=33097 max_rank=7 min=30471 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25891 max_load_reduction_pct=0.438920 moved_to_replicas=147815 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] avg=32768.0000 max=33096 max_rank=0 min=30608 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17817 max_load_reduction_pct=0.349950 moved_to_replicas=99788 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[15517, 6296, 56463, 37805, 58988, 49310, 3281, 34484] avg=32768.0000 max=58988 max_rank=4 min=3281 min_rank=6 imbalance_ratio=1.800171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[50913, 44241, 30608, 29223, 47040, 11147, 22660, 26312] avg=32768.0000 max=50913 max_rank=0 min=11147 min_rank=5 imbalance_ratio=1.553741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] avg=32768.0000 max=33097 max_rank=7 min=30471 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25891 max_load_reduction_pct=0.438920 moved_to_replicas=147815 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[50913, 44241, 30608, 29223, 47040, 11147, 22660, 26312] avg=32768.0000 max=50913 max_rank=0 min=11147 min_rank=5 imbalance_ratio=1.553741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[50913, 44241, 30608, 29223, 47040, 11147, 22660, 26312] avg=32768.0000 max=50913 max_rank=0 min=11147 min_rank=5 imbalance_ratio=1.553741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[15517, 6296, 56463, 37805, 58988, 49310, 3281, 34484] avg=32768.0000 max=58988 max_rank=4 min=3281 min_rank=6 imbalance_ratio=1.800171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[50913, 44241, 30608, 29223, 47040, 11147, 22660, 26312] avg=32768.0000 max=50913 max_rank=0 min=11147 min_rank=5 imbalance_ratio=1.553741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] avg=32768.0000 max=33096 max_rank=0 min=30608 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17817 max_load_reduction_pct=0.349950 moved_to_replicas=99788 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] avg=32768.0000 max=33097 max_rank=7 min=30471 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25891 max_load_reduction_pct=0.438920 moved_to_replicas=147815 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[35674, 11577, 69473, 18156, 72896, 6965, 21005, 26398] avg=32768.0000 max=72896 max_rank=4 min=6965 min_rank=5 imbalance_ratio=2.224609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] avg=32768.0000 max=33096 max_rank=0 min=30608 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17817 max_load_reduction_pct=0.349950 moved_to_replicas=99788 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] avg=32768.0000 max=33096 max_rank=0 min=30608 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17817 max_load_reduction_pct=0.349950 moved_to_replicas=99788 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[50913, 44241, 30608, 29223, 47040, 11147, 22660, 26312] avg=32768.0000 max=50913 max_rank=0 min=11147 min_rank=5 imbalance_ratio=1.553741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] avg=32768.0000 max=33097 max_rank=7 min=30471 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25891 max_load_reduction_pct=0.438920 moved_to_replicas=147815 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] avg=32768.0000 max=33096 max_rank=0 min=30608 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17817 max_load_reduction_pct=0.349950 moved_to_replicas=99788 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39799 max_load_reduction_pct=0.545970 moved_to_replicas=156986 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[35674, 11577, 69473, 18156, 72896, 6965, 21005, 26398] avg=32768.0000 max=72896 max_rank=4 min=6965 min_rank=5 imbalance_ratio=2.224609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[50913, 44241, 30608, 29223, 47040, 11147, 22660, 26312] avg=32768.0000 max=50913 max_rank=0 min=11147 min_rank=5 imbalance_ratio=1.553741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] avg=32768.0000 max=33096 max_rank=0 min=30608 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17817 max_load_reduction_pct=0.349950 moved_to_replicas=99788 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[35674, 11577, 69473, 18156, 72896, 6965, 21005, 26398] avg=32768.0000 max=72896 max_rank=4 min=6965 min_rank=5 imbalance_ratio=2.224609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[35674, 11577, 69473, 18156, 72896, 6965, 21005, 26398] avg=32768.0000 max=72896 max_rank=4 min=6965 min_rank=5 imbalance_ratio=2.224609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[35674, 11577, 69473, 18156, 72896, 6965, 21005, 26398] avg=32768.0000 max=72896 max_rank=4 min=6965 min_rank=5 imbalance_ratio=2.224609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[38276, 27297, 38561, 22228, 26868, 35436, 22536, 50942] avg=32768.0000 max=50942 max_rank=7 min=22228 min_rank=3 imbalance_ratio=1.554626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[50913, 44241, 30608, 29223, 47040, 11147, 22660, 26312] avg=32768.0000 max=50913 max_rank=0 min=11147 min_rank=5 imbalance_ratio=1.553741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39799 max_load_reduction_pct=0.545970 moved_to_replicas=156986 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] avg=32768.0000 max=33096 max_rank=0 min=30608 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17817 max_load_reduction_pct=0.349950 moved_to_replicas=99788 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39799 max_load_reduction_pct=0.545970 moved_to_replicas=156986 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39799 max_load_reduction_pct=0.545970 moved_to_replicas=156986 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39799 max_load_reduction_pct=0.545970 moved_to_replicas=156986 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] avg=32768.0000 max=33097 max_rank=5 min=31647 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17845 max_load_reduction_pct=0.350300 moved_to_replicas=50198 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[35674, 11577, 69473, 18156, 72896, 6965, 21005, 26398] avg=32768.0000 max=72896 max_rank=4 min=6965 min_rank=5 imbalance_ratio=2.224609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] avg=32768.0000 max=33096 max_rank=0 min=30608 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17817 max_load_reduction_pct=0.349950 moved_to_replicas=99788 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[38276, 27297, 38561, 22228, 26868, 35436, 22536, 50942] avg=32768.0000 max=50942 max_rank=7 min=22228 min_rank=3 imbalance_ratio=1.554626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[35674, 11577, 69473, 18156, 72896, 6965, 21005, 26398] avg=32768.0000 max=72896 max_rank=4 min=6965 min_rank=5 imbalance_ratio=2.224609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[38276, 27297, 38561, 22228, 26868, 35436, 22536, 50942] avg=32768.0000 max=50942 max_rank=7 min=22228 min_rank=3 imbalance_ratio=1.554626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39799 max_load_reduction_pct=0.545970 moved_to_replicas=156986 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[38276, 27297, 38561, 22228, 26868, 35436, 22536, 50942] avg=32768.0000 max=50942 max_rank=7 min=22228 min_rank=3 imbalance_ratio=1.554626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[38276, 27297, 38561, 22228, 26868, 35436, 22536, 50942] avg=32768.0000 max=50942 max_rank=7 min=22228 min_rank=3 imbalance_ratio=1.554626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[16478, 69936, 32899, 43540, 45271, 17744, 13205, 23071] avg=32768.0000 max=69936 max_rank=1 min=13205 min_rank=6 imbalance_ratio=2.134277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] avg=32768.0000 max=33097 max_rank=5 min=31647 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17845 max_load_reduction_pct=0.350300 moved_to_replicas=50198 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39799 max_load_reduction_pct=0.545970 moved_to_replicas=156986 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[35674, 11577, 69473, 18156, 72896, 6965, 21005, 26398] avg=32768.0000 max=72896 max_rank=4 min=6965 min_rank=5 imbalance_ratio=2.224609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] avg=32768.0000 max=33097 max_rank=5 min=31647 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17845 max_load_reduction_pct=0.350300 moved_to_replicas=50198 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] avg=32768.0000 max=33097 max_rank=5 min=31647 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17845 max_load_reduction_pct=0.350300 moved_to_replicas=50198 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] avg=32768.0000 max=33097 max_rank=5 min=31647 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17845 max_load_reduction_pct=0.350300 moved_to_replicas=50198 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30669 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=36839 max_load_reduction_pct=0.526753 moved_to_replicas=116389 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[38276, 27297, 38561, 22228, 26868, 35436, 22536, 50942] avg=32768.0000 max=50942 max_rank=7 min=22228 min_rank=3 imbalance_ratio=1.554626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39799 max_load_reduction_pct=0.545970 moved_to_replicas=156986 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[38276, 27297, 38561, 22228, 26868, 35436, 22536, 50942] avg=32768.0000 max=50942 max_rank=7 min=22228 min_rank=3 imbalance_ratio=1.554626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[16478, 69936, 32899, 43540, 45271, 17744, 13205, 23071] avg=32768.0000 max=69936 max_rank=1 min=13205 min_rank=6 imbalance_ratio=2.134277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[16478, 69936, 32899, 43540, 45271, 17744, 13205, 23071] avg=32768.0000 max=69936 max_rank=1 min=13205 min_rank=6 imbalance_ratio=2.134277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] avg=32768.0000 max=33097 max_rank=5 min=31647 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17845 max_load_reduction_pct=0.350300 moved_to_replicas=50198 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[16478, 69936, 32899, 43540, 45271, 17744, 13205, 23071] avg=32768.0000 max=69936 max_rank=1 min=13205 min_rank=6 imbalance_ratio=2.134277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[48930, 21595, 8342, 37564, 25037, 43589, 34318, 42769] avg=32768.0000 max=48930 max_rank=0 min=8342 min_rank=2 imbalance_ratio=1.493225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] avg=32768.0000 max=33097 max_rank=5 min=31647 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17845 max_load_reduction_pct=0.350300 moved_to_replicas=50198 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30669 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=36839 max_load_reduction_pct=0.526753 moved_to_replicas=116389 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[16478, 69936, 32899, 43540, 45271, 17744, 13205, 23071] avg=32768.0000 max=69936 max_rank=1 min=13205 min_rank=6 imbalance_ratio=2.134277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[38276, 27297, 38561, 22228, 26868, 35436, 22536, 50942] avg=32768.0000 max=50942 max_rank=7 min=22228 min_rank=3 imbalance_ratio=1.554626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30669 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=36839 max_load_reduction_pct=0.526753 moved_to_replicas=116389 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31480 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15833 max_load_reduction_pct=0.323585 moved_to_replicas=97276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30669 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=36839 max_load_reduction_pct=0.526753 moved_to_replicas=116389 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[16478, 69936, 32899, 43540, 45271, 17744, 13205, 23071] avg=32768.0000 max=69936 max_rank=1 min=13205 min_rank=6 imbalance_ratio=2.134277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30669 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=36839 max_load_reduction_pct=0.526753 moved_to_replicas=116389 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] avg=32768.0000 max=33097 max_rank=5 min=31647 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17845 max_load_reduction_pct=0.350300 moved_to_replicas=50198 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[48930, 21595, 8342, 37564, 25037, 43589, 34318, 42769] avg=32768.0000 max=48930 max_rank=0 min=8342 min_rank=2 imbalance_ratio=1.493225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[16478, 69936, 32899, 43540, 45271, 17744, 13205, 23071] avg=32768.0000 max=69936 max_rank=1 min=13205 min_rank=6 imbalance_ratio=2.134277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[48930, 21595, 8342, 37564, 25037, 43589, 34318, 42769] avg=32768.0000 max=48930 max_rank=0 min=8342 min_rank=2 imbalance_ratio=1.493225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30669 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=36839 max_load_reduction_pct=0.526753 moved_to_replicas=116389 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[48930, 21595, 8342, 37564, 25037, 43589, 34318, 42769] avg=32768.0000 max=48930 max_rank=0 min=8342 min_rank=2 imbalance_ratio=1.493225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31480 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15833 max_load_reduction_pct=0.323585 moved_to_replicas=97276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31480 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15833 max_load_reduction_pct=0.323585 moved_to_replicas=97276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[48930, 21595, 8342, 37564, 25037, 43589, 34318, 42769] avg=32768.0000 max=48930 max_rank=0 min=8342 min_rank=2 imbalance_ratio=1.493225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[16478, 69936, 32899, 43540, 45271, 17744, 13205, 23071] avg=32768.0000 max=69936 max_rank=1 min=13205 min_rank=6 imbalance_ratio=2.134277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30669 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=36839 max_load_reduction_pct=0.526753 moved_to_replicas=116389 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31480 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15833 max_load_reduction_pct=0.323585 moved_to_replicas=97276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[48930, 21595, 8342, 37564, 25037, 43589, 34318, 42769] avg=32768.0000 max=48930 max_rank=0 min=8342 min_rank=2 imbalance_ratio=1.493225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31480 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15833 max_load_reduction_pct=0.323585 moved_to_replicas=97276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30669 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=36839 max_load_reduction_pct=0.526753 moved_to_replicas=116389 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[48930, 21595, 8342, 37564, 25037, 43589, 34318, 42769] avg=32768.0000 max=48930 max_rank=0 min=8342 min_rank=2 imbalance_ratio=1.493225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31480 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15833 max_load_reduction_pct=0.323585 moved_to_replicas=97276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[48930, 21595, 8342, 37564, 25037, 43589, 34318, 42769] avg=32768.0000 max=48930 max_rank=0 min=8342 min_rank=2 imbalance_ratio=1.493225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31480 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15833 max_load_reduction_pct=0.323585 moved_to_replicas=97276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31480 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15833 max_load_reduction_pct=0.323585 moved_to_replicas=97276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=32 total_expected_recv=1047457 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56192 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=32 total_expected_recv=1046435 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57472 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=32 total_expected_recv=1047283 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57216 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=32 total_expected_recv=1053025 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=50944 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=32 total_expected_recv=1050291 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=53760 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=32 total_expected_recv=1047634 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=55808 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=32 total_expected_recv=1047482 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56448 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=32 total_expected_recv=1049001 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=54912 peak_demand=33097 required_capacity=1.010040 + [2026-08-30 05:11:41] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 1101.6 | throughput per GPU (TFLOP/s/GPU): 302.6 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101621E+01 | loss scale: 1.0 | grad norm: 45.875 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=34816 expected_recv=65864 expected_overflow=0 exact_recv_per_rank=[32989, 33096, 31888, 33091, 33096, 33095, 31793, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32989, 33096, 31888, 33091, 33096, 33095, 31793, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=34816 expected_recv=66085 expected_overflow=0 exact_recv_per_rank=[32989, 33096, 31888, 33091, 33096, 33095, 31793, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=34816 expected_recv=63968 expected_overflow=0 exact_recv_per_rank=[32989, 33096, 31888, 33091, 33096, 33095, 31793, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=34816 expected_recv=66187 expected_overflow=0 exact_recv_per_rank=[32989, 33096, 31888, 33091, 33096, 33095, 31793, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=34816 expected_recv=63609 expected_overflow=0 exact_recv_per_rank=[32989, 33096, 31888, 33091, 33096, 33095, 31793, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32989, 33096, 31888, 33091, 33096, 33095, 31793, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32989, 33096, 31888, 33091, 33096, 33095, 31793, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=34816 expected_recv=65864 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=34816 expected_recv=66085 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=34816 expected_recv=63968 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=34816 expected_recv=66187 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=34816 expected_recv=63609 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=64896 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=64384 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 32409, 31158, 33095, 33096, 33097, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=34816 expected_recv=65505 expected_overflow=0 exact_recv_per_rank=[33096, 32409, 31158, 33095, 33096, 33097, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=34816 expected_recv=62338 expected_overflow=0 exact_recv_per_rank=[33096, 32409, 31158, 33095, 33096, 33097, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32409, 31158, 33095, 33096, 33097, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=34816 expected_recv=65505 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=34816 expected_recv=62338 actual_valid_recv=62976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=34816 expected_recv=65483 expected_overflow=0 exact_recv_per_rank=[33096, 32409, 31158, 33095, 33096, 33097, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 32409, 31158, 33095, 33096, 33097, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32409, 31158, 33095, 33096, 33097, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=62976 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32409, 31158, 33095, 33096, 33097, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=34816 expected_recv=65483 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32881, 31597, 32186, 33096, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=34816 expected_recv=64694 expected_overflow=0 exact_recv_per_rank=[32881, 31597, 32186, 33096, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=34816 expected_recv=65977 expected_overflow=0 exact_recv_per_rank=[32881, 31597, 32186, 33096, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=34816 expected_recv=64694 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32881, 31597, 32186, 33096, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32881, 31597, 32186, 33096, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=34816 expected_recv=65977 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=34816 expected_recv=64500 expected_overflow=0 exact_recv_per_rank=[32881, 31597, 32186, 33096, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=34816 expected_recv=65846 expected_overflow=0 exact_recv_per_rank=[32881, 31597, 32186, 33096, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=34816 expected_recv=64500 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=34816 expected_recv=65846 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=34816 expected_recv=65955 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 30710, 32857, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=34816 expected_recv=65955 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=34816 expected_recv=64694 expected_overflow=0 exact_recv_per_rank=[32881, 31597, 32186, 33096, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 30710, 32857, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 30710, 32857, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=34816 expected_recv=64694 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 30710, 32857, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=34816 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 30710, 32857, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=34816 expected_recv=63806 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 30710, 32857, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 30710, 32857, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=34816 expected_recv=63568 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=34816 expected_recv=63806 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=64384 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=34816 expected_recv=65724 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 30942, 33096, 33094, 32628, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=64640 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=34816 expected_recv=65724 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=34816 expected_recv=63367 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 30942, 33096, 33094, 32628, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 30942, 33096, 33094, 32628, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=34816 expected_recv=63367 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 30942, 33096, 33094, 32628, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=34816 expected_recv=64238 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 30942, 33096, 33094, 32628, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 30942, 33096, 33094, 32628, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 30942, 33096, 33094, 32628, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=64256 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=34816 expected_recv=64238 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32160, 33095, 33096, 31408, 33097, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65024 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 33096, 30710, 32857, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32160, 33095, 33096, 31408, 33097, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=34816 expected_recv=64188 expected_overflow=0 exact_recv_per_rank=[32160, 33095, 33096, 31408, 33097, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=34816 expected_recv=64188 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=34816 expected_recv=66195 expected_overflow=0 exact_recv_per_rank=[32160, 33095, 33096, 31408, 33097, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32160, 33095, 33096, 31408, 33097, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32160, 33095, 33096, 31408, 33097, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65024 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=34816 expected_recv=66195 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=34816 expected_recv=62948 expected_overflow=0 exact_recv_per_rank=[32160, 33095, 33096, 31408, 33097, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=34816 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[32299, 33096, 33096, 33096, 33096, 33096, 32142, 32223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=34816 expected_recv=62948 actual_valid_recv=63744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=34816 expected_recv=66194 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 30942, 33096, 33094, 32628, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=63744 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=67200 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=34816 expected_recv=64570 expected_overflow=0 exact_recv_per_rank=[32299, 33096, 33096, 33096, 33096, 33096, 32142, 32223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=34816 expected_recv=64807 expected_overflow=0 exact_recv_per_rank=[32299, 33096, 33096, 33096, 33096, 33096, 32142, 32223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=34816 expected_recv=64570 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=34816 expected_recv=64807 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32299, 33096, 33096, 33096, 33096, 33096, 32142, 32223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=34816 expected_recv=65238 expected_overflow=0 exact_recv_per_rank=[32299, 33096, 33096, 33096, 33096, 33096, 32142, 32223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=34816 expected_recv=64905 expected_overflow=0 exact_recv_per_rank=[32299, 33096, 33096, 33096, 33096, 33096, 32142, 32223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=34816 expected_recv=65238 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32299, 33096, 33096, 33096, 33096, 33096, 32142, 32223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30502, 33096, 33096, 33066, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=34816 expected_recv=64905 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=34816 expected_recv=66189 expected_overflow=0 exact_recv_per_rank=[32160, 33095, 33096, 31408, 33097, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=34816 expected_recv=63313 expected_overflow=0 exact_recv_per_rank=[30502, 33096, 33096, 33066, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=34816 expected_recv=65881 expected_overflow=0 exact_recv_per_rank=[30502, 33096, 33096, 33066, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=34816 expected_recv=66189 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=34816 expected_recv=63313 actual_valid_recv=64128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=34816 expected_recv=65881 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=64128 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30502, 33096, 33096, 33066, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30502, 33096, 33096, 33066, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30502, 33096, 33096, 33066, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 30472, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=34816 expected_recv=64134 expected_overflow=0 exact_recv_per_rank=[30502, 33096, 33096, 33066, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=67200 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=34816 expected_recv=64134 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32299, 33096, 33096, 33096, 33096, 33096, 32142, 32223] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 30472, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=64896 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 30472, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 30472, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=34816 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 30472, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 30472, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32327, 33097, 31489, 33096, 33096, 32848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=34816 expected_recv=63568 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=34816 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 30472, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=64384 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=34816 expected_recv=63568 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30502, 33096, 33096, 33066, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32327, 33097, 31489, 33096, 33096, 32848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=34816 expected_recv=65422 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32327, 33097, 31489, 33096, 33096, 32848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=64256 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=34816 expected_recv=63182 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32327, 33097, 31489, 33096, 33096, 32848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=34816 expected_recv=65422 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32327, 33097, 31489, 33096, 33096, 32848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=34816 expected_recv=63182 actual_valid_recv=64128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=64128 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=34816 expected_recv=65944 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32327, 33097, 31489, 33096, 33096, 32848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=34816 expected_recv=64973 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32327, 33097, 31489, 33096, 33096, 32848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=34816 expected_recv=64691 expected_overflow=0 exact_recv_per_rank=[33096, 31398, 33096, 33096, 33095, 32171, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=34816 expected_recv=65944 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=34816 expected_recv=64973 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=34816 expected_recv=64691 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33097, 30472, 33097, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=34816 expected_recv=65611 expected_overflow=0 exact_recv_per_rank=[33096, 31398, 33096, 33096, 33095, 32171, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31398, 33096, 33096, 33095, 32171, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=34816 expected_recv=65611 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 31398, 33096, 33096, 33095, 32171, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 31398, 33096, 33096, 33095, 32171, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=34816 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33096, 31398, 33096, 33096, 33095, 32171, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31398, 33096, 33096, 33095, 32171, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=34816 expected_recv=66097 expected_overflow=0 exact_recv_per_rank=[31525, 33096, 33095, 33096, 33096, 33001, 32139, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=34816 expected_recv=66194 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=34816 expected_recv=66097 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32327, 33097, 31489, 33096, 33096, 32848] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=34816 expected_recv=64621 expected_overflow=0 exact_recv_per_rank=[31525, 33096, 33095, 33096, 33096, 33001, 32139, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=34816 expected_recv=65313 expected_overflow=0 exact_recv_per_rank=[31525, 33096, 33095, 33096, 33096, 33001, 32139, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=34816 expected_recv=64621 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=34816 expected_recv=65313 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31525, 33096, 33095, 33096, 33096, 33001, 32139, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=34816 expected_recv=63491 expected_overflow=0 exact_recv_per_rank=[31525, 33096, 33095, 33096, 33096, 33001, 32139, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=34816 expected_recv=63491 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=34816 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[31525, 33096, 33095, 33096, 33096, 33001, 32139, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31525, 33096, 33095, 33096, 33096, 33001, 32139, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=34816 expected_recv=65478 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=64256 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=34816 expected_recv=66190 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=34816 expected_recv=65478 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=34816 expected_recv=65884 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=34816 expected_recv=63026 expected_overflow=0 exact_recv_per_rank=[33096, 31398, 33096, 33096, 33095, 32171, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=34816 expected_recv=65884 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=34816 expected_recv=63026 actual_valid_recv=64128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=64128 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=34816 expected_recv=61967 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32939, 33096, 33095, 30630, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=34816 expected_recv=61967 actual_valid_recv=63104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=63104 slack_slots=6528 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=34816 expected_recv=66035 expected_overflow=0 exact_recv_per_rank=[32939, 33096, 33095, 30630, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31525, 33096, 33095, 33096, 33096, 33001, 32139, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32939, 33096, 33095, 30630, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=34816 expected_recv=66035 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32939, 33096, 33095, 30630, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32939, 33096, 33095, 30630, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32939, 33096, 33095, 30630, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30798, 33095, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=34816 expected_recv=61103 expected_overflow=0 exact_recv_per_rank=[32939, 33096, 33095, 30630, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=34816 expected_recv=61103 actual_valid_recv=62336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=62336 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30798, 33095, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=34816 expected_recv=65932 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30798, 33095, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=34816 expected_recv=65932 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=34816 expected_recv=65867 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30798, 33095, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=34816 expected_recv=61531 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30798, 33095, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=34816 expected_recv=65929 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32671, 33098, 32721, 33095, 31272, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30798, 33095, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=34816 expected_recv=65867 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=34816 expected_recv=61531 actual_valid_recv=62464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30798, 33095, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=34816 expected_recv=65929 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=62464 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32671, 33098, 32721, 33095, 31272, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32939, 33096, 33095, 30630, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=34816 expected_recv=64369 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32671, 33098, 32721, 33095, 31272, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=67200 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=34816 expected_recv=64369 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=34816 expected_recv=63970 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32671, 33098, 32721, 33095, 31272, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=34816 expected_recv=65519 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32671, 33098, 32721, 33095, 31272, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32671, 33098, 32721, 33095, 31272, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=34816 expected_recv=63970 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=34816 expected_recv=65519 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=34816 expected_recv=65925 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32671, 33098, 32721, 33095, 31272, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[32989, 30679, 31181, 17809, 38492, 48377, 26397, 36220] avg=32768.0000 max=48377 max_rank=5 min=17809 min_rank=3 imbalance_ratio=1.476349 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=64640 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=34816 expected_recv=65925 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30798, 33095, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[32989, 30679, 31181, 17809, 38492, 48377, 26397, 36220] avg=32768.0000 max=48377 max_rank=5 min=17809 min_rank=3 imbalance_ratio=1.476349 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[32989, 30679, 31181, 17809, 38492, 48377, 26397, 36220] avg=32768.0000 max=48377 max_rank=5 min=17809 min_rank=3 imbalance_ratio=1.476349 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[32989, 33096, 31888, 33091, 33096, 33095, 31793, 33096] avg=32768.0000 max=33096 max_rank=1 min=31793 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15281 max_load_reduction_pct=0.315873 moved_to_replicas=45444 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[32989, 33096, 31888, 33091, 33096, 33095, 31793, 33096] avg=32768.0000 max=33096 max_rank=1 min=31793 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15281 max_load_reduction_pct=0.315873 moved_to_replicas=45444 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[32989, 30679, 31181, 17809, 38492, 48377, 26397, 36220] avg=32768.0000 max=48377 max_rank=5 min=17809 min_rank=3 imbalance_ratio=1.476349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[32989, 30679, 31181, 17809, 38492, 48377, 26397, 36220] avg=32768.0000 max=48377 max_rank=5 min=17809 min_rank=3 imbalance_ratio=1.476349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[32989, 33096, 31888, 33091, 33096, 33095, 31793, 33096] avg=32768.0000 max=33096 max_rank=1 min=31793 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15281 max_load_reduction_pct=0.315873 moved_to_replicas=45444 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[32989, 30679, 31181, 17809, 38492, 48377, 26397, 36220] avg=32768.0000 max=48377 max_rank=5 min=17809 min_rank=3 imbalance_ratio=1.476349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[32989, 30679, 31181, 17809, 38492, 48377, 26397, 36220] avg=32768.0000 max=48377 max_rank=5 min=17809 min_rank=3 imbalance_ratio=1.476349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[42829, 14620, 237, 251, 2418, 37059, 88373, 76357] avg=32768.0000 max=88373 max_rank=6 min=237 min_rank=2 imbalance_ratio=2.696930 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32671, 33098, 32721, 33095, 31272, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[32989, 33096, 31888, 33091, 33096, 33095, 31793, 33096] avg=32768.0000 max=33096 max_rank=1 min=31793 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15281 max_load_reduction_pct=0.315873 moved_to_replicas=45444 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[32989, 33096, 31888, 33091, 33096, 33095, 31793, 33096] avg=32768.0000 max=33096 max_rank=1 min=31793 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15281 max_load_reduction_pct=0.315873 moved_to_replicas=45444 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[32989, 33096, 31888, 33091, 33096, 33095, 31793, 33096] avg=32768.0000 max=33096 max_rank=1 min=31793 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15281 max_load_reduction_pct=0.315873 moved_to_replicas=45444 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[33096, 32409, 31158, 33095, 33096, 33097, 33097, 33096] avg=32768.0000 max=33097 max_rank=5 min=31158 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55276 max_load_reduction_pct=0.625485 moved_to_replicas=215025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[32989, 33096, 31888, 33091, 33096, 33095, 31793, 33096] avg=32768.0000 max=33096 max_rank=1 min=31793 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15281 max_load_reduction_pct=0.315873 moved_to_replicas=45444 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[42829, 14620, 237, 251, 2418, 37059, 88373, 76357] avg=32768.0000 max=88373 max_rank=6 min=237 min_rank=2 imbalance_ratio=2.696930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[42829, 14620, 237, 251, 2418, 37059, 88373, 76357] avg=32768.0000 max=88373 max_rank=6 min=237 min_rank=2 imbalance_ratio=2.696930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[42829, 14620, 237, 251, 2418, 37059, 88373, 76357] avg=32768.0000 max=88373 max_rank=6 min=237 min_rank=2 imbalance_ratio=2.696930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[42829, 14620, 237, 251, 2418, 37059, 88373, 76357] avg=32768.0000 max=88373 max_rank=6 min=237 min_rank=2 imbalance_ratio=2.696930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[33096, 32409, 31158, 33095, 33096, 33097, 33097, 33096] avg=32768.0000 max=33097 max_rank=5 min=31158 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55276 max_load_reduction_pct=0.625485 moved_to_replicas=215025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[33096, 32409, 31158, 33095, 33096, 33097, 33097, 33096] avg=32768.0000 max=33097 max_rank=5 min=31158 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55276 max_load_reduction_pct=0.625485 moved_to_replicas=215025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[30965, 31597, 32186, 41657, 30256, 48305, 45170, 2008] avg=32768.0000 max=48305 max_rank=5 min=2008 min_rank=7 imbalance_ratio=1.474152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[42829, 14620, 237, 251, 2418, 37059, 88373, 76357] avg=32768.0000 max=88373 max_rank=6 min=237 min_rank=2 imbalance_ratio=2.696930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[42829, 14620, 237, 251, 2418, 37059, 88373, 76357] avg=32768.0000 max=88373 max_rank=6 min=237 min_rank=2 imbalance_ratio=2.696930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[32989, 30679, 31181, 17809, 38492, 48377, 26397, 36220] avg=32768.0000 max=48377 max_rank=5 min=17809 min_rank=3 imbalance_ratio=1.476349 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[33096, 32409, 31158, 33095, 33096, 33097, 33097, 33096] avg=32768.0000 max=33097 max_rank=5 min=31158 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55276 max_load_reduction_pct=0.625485 moved_to_replicas=215025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[33096, 32409, 31158, 33095, 33096, 33097, 33097, 33096] avg=32768.0000 max=33097 max_rank=5 min=31158 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55276 max_load_reduction_pct=0.625485 moved_to_replicas=215025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[32881, 31597, 32186, 33096, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=31597 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=15208 max_load_reduction_pct=0.314833 moved_to_replicas=69054 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[30965, 31597, 32186, 41657, 30256, 48305, 45170, 2008] avg=32768.0000 max=48305 max_rank=5 min=2008 min_rank=7 imbalance_ratio=1.474152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[33096, 32409, 31158, 33095, 33096, 33097, 33097, 33096] avg=32768.0000 max=33097 max_rank=5 min=31158 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55276 max_load_reduction_pct=0.625485 moved_to_replicas=215025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[33096, 32409, 31158, 33095, 33096, 33097, 33097, 33096] avg=32768.0000 max=33097 max_rank=5 min=31158 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55276 max_load_reduction_pct=0.625485 moved_to_replicas=215025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[30965, 31597, 32186, 41657, 30256, 48305, 45170, 2008] avg=32768.0000 max=48305 max_rank=5 min=2008 min_rank=7 imbalance_ratio=1.474152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[32989, 33096, 31888, 33091, 33096, 33095, 31793, 33096] avg=32768.0000 max=33096 max_rank=1 min=31793 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=15281 max_load_reduction_pct=0.315873 moved_to_replicas=45444 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[30965, 31597, 32186, 41657, 30256, 48305, 45170, 2008] avg=32768.0000 max=48305 max_rank=5 min=2008 min_rank=7 imbalance_ratio=1.474152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[30965, 31597, 32186, 41657, 30256, 48305, 45170, 2008] avg=32768.0000 max=48305 max_rank=5 min=2008 min_rank=7 imbalance_ratio=1.474152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[32881, 31597, 32186, 33096, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=31597 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=15208 max_load_reduction_pct=0.314833 moved_to_replicas=69054 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[32881, 31597, 32186, 33096, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=31597 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=15208 max_load_reduction_pct=0.314833 moved_to_replicas=69054 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[44540, 29916, 44371, 38883, 787, 513, 30550, 72584] avg=32768.0000 max=72584 max_rank=7 min=513 min_rank=5 imbalance_ratio=2.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[30965, 31597, 32186, 41657, 30256, 48305, 45170, 2008] avg=32768.0000 max=48305 max_rank=5 min=2008 min_rank=7 imbalance_ratio=1.474152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[32881, 31597, 32186, 33096, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=31597 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=15208 max_load_reduction_pct=0.314833 moved_to_replicas=69054 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[32881, 31597, 32186, 33096, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=31597 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=15208 max_load_reduction_pct=0.314833 moved_to_replicas=69054 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[42829, 14620, 237, 251, 2418, 37059, 88373, 76357] avg=32768.0000 max=88373 max_rank=6 min=237 min_rank=2 imbalance_ratio=2.696930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[30965, 31597, 32186, 41657, 30256, 48305, 45170, 2008] avg=32768.0000 max=48305 max_rank=5 min=2008 min_rank=7 imbalance_ratio=1.474152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 30710, 32857, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30710 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=39487 max_load_reduction_pct=0.544018 moved_to_replicas=142626 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[44540, 29916, 44371, 38883, 787, 513, 30550, 72584] avg=32768.0000 max=72584 max_rank=7 min=513 min_rank=5 imbalance_ratio=2.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[32881, 31597, 32186, 33096, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=31597 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=15208 max_load_reduction_pct=0.314833 moved_to_replicas=69054 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[44540, 29916, 44371, 38883, 787, 513, 30550, 72584] avg=32768.0000 max=72584 max_rank=7 min=513 min_rank=5 imbalance_ratio=2.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[33096, 32409, 31158, 33095, 33096, 33097, 33097, 33096] avg=32768.0000 max=33097 max_rank=5 min=31158 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55276 max_load_reduction_pct=0.625485 moved_to_replicas=215025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[44540, 29916, 44371, 38883, 787, 513, 30550, 72584] avg=32768.0000 max=72584 max_rank=7 min=513 min_rank=5 imbalance_ratio=2.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[32881, 31597, 32186, 33096, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=31597 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=15208 max_load_reduction_pct=0.314833 moved_to_replicas=69054 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[44540, 29916, 44371, 38883, 787, 513, 30550, 72584] avg=32768.0000 max=72584 max_rank=7 min=513 min_rank=5 imbalance_ratio=2.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 30710, 32857, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30710 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=39487 max_load_reduction_pct=0.544018 moved_to_replicas=142626 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[4514, 8166, 20161, 95922, 7289, 32628, 54500, 38964] avg=32768.0000 max=95922 max_rank=3 min=4514 min_rank=0 imbalance_ratio=2.927307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 30710, 32857, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30710 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=39487 max_load_reduction_pct=0.544018 moved_to_replicas=142626 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[44540, 29916, 44371, 38883, 787, 513, 30550, 72584] avg=32768.0000 max=72584 max_rank=7 min=513 min_rank=5 imbalance_ratio=2.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 30710, 32857, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30710 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=39487 max_load_reduction_pct=0.544018 moved_to_replicas=142626 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 30710, 32857, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30710 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=39487 max_load_reduction_pct=0.544018 moved_to_replicas=142626 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[30965, 31597, 32186, 41657, 30256, 48305, 45170, 2008] avg=32768.0000 max=48305 max_rank=5 min=2008 min_rank=7 imbalance_ratio=1.474152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[33096, 33097, 30942, 33096, 33094, 32628, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=30942 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62825 max_load_reduction_pct=0.654959 moved_to_replicas=175583 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[44540, 29916, 44371, 38883, 787, 513, 30550, 72584] avg=32768.0000 max=72584 max_rank=7 min=513 min_rank=5 imbalance_ratio=2.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[4514, 8166, 20161, 95922, 7289, 32628, 54500, 38964] avg=32768.0000 max=95922 max_rank=3 min=4514 min_rank=0 imbalance_ratio=2.927307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 30710, 32857, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30710 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=39487 max_load_reduction_pct=0.544018 moved_to_replicas=142626 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[4514, 8166, 20161, 95922, 7289, 32628, 54500, 38964] avg=32768.0000 max=95922 max_rank=3 min=4514 min_rank=0 imbalance_ratio=2.927307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[32881, 31597, 32186, 33096, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=31597 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=15208 max_load_reduction_pct=0.314833 moved_to_replicas=69054 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[4514, 8166, 20161, 95922, 7289, 32628, 54500, 38964] avg=32768.0000 max=95922 max_rank=3 min=4514 min_rank=0 imbalance_ratio=2.927307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 30710, 32857, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30710 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=39487 max_load_reduction_pct=0.544018 moved_to_replicas=142626 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[33096, 33097, 30942, 33096, 33094, 32628, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=30942 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62825 max_load_reduction_pct=0.654959 moved_to_replicas=175583 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[4514, 8166, 20161, 95922, 7289, 32628, 54500, 38964] avg=32768.0000 max=95922 max_rank=3 min=4514 min_rank=0 imbalance_ratio=2.927307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[28432, 4295, 34315, 1699, 70882, 17984, 45288, 59249] avg=32768.0000 max=70882 max_rank=4 min=1699 min_rank=3 imbalance_ratio=2.163147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[33096, 33097, 30942, 33096, 33094, 32628, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=30942 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62825 max_load_reduction_pct=0.654959 moved_to_replicas=175583 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[33096, 33097, 30942, 33096, 33094, 32628, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=30942 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62825 max_load_reduction_pct=0.654959 moved_to_replicas=175583 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[4514, 8166, 20161, 95922, 7289, 32628, 54500, 38964] avg=32768.0000 max=95922 max_rank=3 min=4514 min_rank=0 imbalance_ratio=2.927307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[32160, 33095, 33096, 31408, 33097, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31408 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=37785 max_load_reduction_pct=0.533069 moved_to_replicas=149582 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[33096, 33097, 30942, 33096, 33094, 32628, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=30942 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62825 max_load_reduction_pct=0.654959 moved_to_replicas=175583 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[44540, 29916, 44371, 38883, 787, 513, 30550, 72584] avg=32768.0000 max=72584 max_rank=7 min=513 min_rank=5 imbalance_ratio=2.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[4514, 8166, 20161, 95922, 7289, 32628, 54500, 38964] avg=32768.0000 max=95922 max_rank=3 min=4514 min_rank=0 imbalance_ratio=2.927307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[28432, 4295, 34315, 1699, 70882, 17984, 45288, 59249] avg=32768.0000 max=70882 max_rank=4 min=1699 min_rank=3 imbalance_ratio=2.163147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[28432, 4295, 34315, 1699, 70882, 17984, 45288, 59249] avg=32768.0000 max=70882 max_rank=4 min=1699 min_rank=3 imbalance_ratio=2.163147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[33096, 33097, 30942, 33096, 33094, 32628, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=30942 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62825 max_load_reduction_pct=0.654959 moved_to_replicas=175583 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[28432, 4295, 34315, 1699, 70882, 17984, 45288, 59249] avg=32768.0000 max=70882 max_rank=4 min=1699 min_rank=3 imbalance_ratio=2.163147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[33097, 33096, 33096, 33096, 30710, 32857, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30710 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=39487 max_load_reduction_pct=0.544018 moved_to_replicas=142626 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[24086, 33575, 9470, 58333, 53772, 18543, 32142, 32223] avg=32768.0000 max=58333 max_rank=3 min=9470 min_rank=2 imbalance_ratio=1.780182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[32160, 33095, 33096, 31408, 33097, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31408 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=37785 max_load_reduction_pct=0.533069 moved_to_replicas=149582 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[33096, 33097, 30942, 33096, 33094, 32628, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=30942 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62825 max_load_reduction_pct=0.654959 moved_to_replicas=175583 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[32160, 33095, 33096, 31408, 33097, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31408 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=37785 max_load_reduction_pct=0.533069 moved_to_replicas=149582 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[28432, 4295, 34315, 1699, 70882, 17984, 45288, 59249] avg=32768.0000 max=70882 max_rank=4 min=1699 min_rank=3 imbalance_ratio=2.163147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[32160, 33095, 33096, 31408, 33097, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31408 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=37785 max_load_reduction_pct=0.533069 moved_to_replicas=149582 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[32299, 33096, 33096, 33096, 33096, 33096, 32142, 32223] avg=32768.0000 max=33096 max_rank=1 min=32142 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25237 max_load_reduction_pct=0.432637 moved_to_replicas=94857 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[28432, 4295, 34315, 1699, 70882, 17984, 45288, 59249] avg=32768.0000 max=70882 max_rank=4 min=1699 min_rank=3 imbalance_ratio=2.163147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[4514, 8166, 20161, 95922, 7289, 32628, 54500, 38964] avg=32768.0000 max=95922 max_rank=3 min=4514 min_rank=0 imbalance_ratio=2.927307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[24086, 33575, 9470, 58333, 53772, 18543, 32142, 32223] avg=32768.0000 max=58333 max_rank=3 min=9470 min_rank=2 imbalance_ratio=1.780182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[28432, 4295, 34315, 1699, 70882, 17984, 45288, 59249] avg=32768.0000 max=70882 max_rank=4 min=1699 min_rank=3 imbalance_ratio=2.163147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[24086, 33575, 9470, 58333, 53772, 18543, 32142, 32223] avg=32768.0000 max=58333 max_rank=3 min=9470 min_rank=2 imbalance_ratio=1.780182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[32160, 33095, 33096, 31408, 33097, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31408 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=37785 max_load_reduction_pct=0.533069 moved_to_replicas=149582 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[32160, 33095, 33096, 31408, 33097, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31408 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=37785 max_load_reduction_pct=0.533069 moved_to_replicas=149582 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[24086, 33575, 9470, 58333, 53772, 18543, 32142, 32223] avg=32768.0000 max=58333 max_rank=3 min=9470 min_rank=2 imbalance_ratio=1.780182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[33096, 33097, 30942, 33096, 33094, 32628, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=30942 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62825 max_load_reduction_pct=0.654959 moved_to_replicas=175583 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[30498, 65460, 1371, 29036, 4507, 65526, 3797, 61949] avg=32768.0000 max=65526 max_rank=5 min=1371 min_rank=2 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[32299, 33096, 33096, 33096, 33096, 33096, 32142, 32223] avg=32768.0000 max=33096 max_rank=1 min=32142 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25237 max_load_reduction_pct=0.432637 moved_to_replicas=94857 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[32160, 33095, 33096, 31408, 33097, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31408 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=37785 max_load_reduction_pct=0.533069 moved_to_replicas=149582 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[32299, 33096, 33096, 33096, 33096, 33096, 32142, 32223] avg=32768.0000 max=33096 max_rank=1 min=32142 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25237 max_load_reduction_pct=0.432637 moved_to_replicas=94857 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[32299, 33096, 33096, 33096, 33096, 33096, 32142, 32223] avg=32768.0000 max=33096 max_rank=1 min=32142 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25237 max_load_reduction_pct=0.432637 moved_to_replicas=94857 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[30502, 33096, 33096, 33066, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30502 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=192212 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[24086, 33575, 9470, 58333, 53772, 18543, 32142, 32223] avg=32768.0000 max=58333 max_rank=3 min=9470 min_rank=2 imbalance_ratio=1.780182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[24086, 33575, 9470, 58333, 53772, 18543, 32142, 32223] avg=32768.0000 max=58333 max_rank=3 min=9470 min_rank=2 imbalance_ratio=1.780182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[28432, 4295, 34315, 1699, 70882, 17984, 45288, 59249] avg=32768.0000 max=70882 max_rank=4 min=1699 min_rank=3 imbalance_ratio=2.163147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[30498, 65460, 1371, 29036, 4507, 65526, 3797, 61949] avg=32768.0000 max=65526 max_rank=5 min=1371 min_rank=2 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[24086, 33575, 9470, 58333, 53772, 18543, 32142, 32223] avg=32768.0000 max=58333 max_rank=3 min=9470 min_rank=2 imbalance_ratio=1.780182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[30498, 65460, 1371, 29036, 4507, 65526, 3797, 61949] avg=32768.0000 max=65526 max_rank=5 min=1371 min_rank=2 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[32299, 33096, 33096, 33096, 33096, 33096, 32142, 32223] avg=32768.0000 max=33096 max_rank=1 min=32142 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25237 max_load_reduction_pct=0.432637 moved_to_replicas=94857 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[30498, 65460, 1371, 29036, 4507, 65526, 3797, 61949] avg=32768.0000 max=65526 max_rank=5 min=1371 min_rank=2 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[32299, 33096, 33096, 33096, 33096, 33096, 32142, 32223] avg=32768.0000 max=33096 max_rank=1 min=32142 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25237 max_load_reduction_pct=0.432637 moved_to_replicas=94857 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[15033, 8682, 3252, 28195, 58170, 80064, 2524, 66224] avg=32768.0000 max=80064 max_rank=5 min=2524 min_rank=6 imbalance_ratio=2.443359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[32160, 33095, 33096, 31408, 33097, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=4 min=31408 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=37785 max_load_reduction_pct=0.533069 moved_to_replicas=149582 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[30502, 33096, 33096, 33066, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30502 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=192212 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[32299, 33096, 33096, 33096, 33096, 33096, 32142, 32223] avg=32768.0000 max=33096 max_rank=1 min=32142 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25237 max_load_reduction_pct=0.432637 moved_to_replicas=94857 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[30502, 33096, 33096, 33066, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30502 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=192212 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[30502, 33096, 33096, 33066, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30502 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=192212 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[33095, 33095, 33097, 30472, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46967 max_load_reduction_pct=0.586618 moved_to_replicas=221374 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[30498, 65460, 1371, 29036, 4507, 65526, 3797, 61949] avg=32768.0000 max=65526 max_rank=5 min=1371 min_rank=2 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[30498, 65460, 1371, 29036, 4507, 65526, 3797, 61949] avg=32768.0000 max=65526 max_rank=5 min=1371 min_rank=2 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[24086, 33575, 9470, 58333, 53772, 18543, 32142, 32223] avg=32768.0000 max=58333 max_rank=3 min=9470 min_rank=2 imbalance_ratio=1.780182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[15033, 8682, 3252, 28195, 58170, 80064, 2524, 66224] avg=32768.0000 max=80064 max_rank=5 min=2524 min_rank=6 imbalance_ratio=2.443359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[30498, 65460, 1371, 29036, 4507, 65526, 3797, 61949] avg=32768.0000 max=65526 max_rank=5 min=1371 min_rank=2 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[15033, 8682, 3252, 28195, 58170, 80064, 2524, 66224] avg=32768.0000 max=80064 max_rank=5 min=2524 min_rank=6 imbalance_ratio=2.443359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[15033, 8682, 3252, 28195, 58170, 80064, 2524, 66224] avg=32768.0000 max=80064 max_rank=5 min=2524 min_rank=6 imbalance_ratio=2.443359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[30502, 33096, 33096, 33066, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30502 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=192212 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[30502, 33096, 33096, 33066, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30502 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=192212 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[64288, 56422, 32327, 11857, 58, 48753, 15591, 32848] avg=32768.0000 max=64288 max_rank=0 min=58 min_rank=4 imbalance_ratio=1.961914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[33095, 33095, 33097, 30472, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46967 max_load_reduction_pct=0.586618 moved_to_replicas=221374 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[30502, 33096, 33096, 33066, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30502 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=192212 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[32299, 33096, 33096, 33096, 33096, 33096, 32142, 32223] avg=32768.0000 max=33096 max_rank=1 min=32142 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25237 max_load_reduction_pct=0.432637 moved_to_replicas=94857 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[33095, 33095, 33097, 30472, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46967 max_load_reduction_pct=0.586618 moved_to_replicas=221374 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[33095, 33095, 33097, 30472, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46967 max_load_reduction_pct=0.586618 moved_to_replicas=221374 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[33096, 33095, 32327, 33097, 31489, 33096, 33096, 32848] avg=32768.0000 max=33097 max_rank=3 min=31489 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31191 max_load_reduction_pct=0.485176 moved_to_replicas=151459 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[15033, 8682, 3252, 28195, 58170, 80064, 2524, 66224] avg=32768.0000 max=80064 max_rank=5 min=2524 min_rank=6 imbalance_ratio=2.443359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[15033, 8682, 3252, 28195, 58170, 80064, 2524, 66224] avg=32768.0000 max=80064 max_rank=5 min=2524 min_rank=6 imbalance_ratio=2.443359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[15033, 8682, 3252, 28195, 58170, 80064, 2524, 66224] avg=32768.0000 max=80064 max_rank=5 min=2524 min_rank=6 imbalance_ratio=2.443359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[64288, 56422, 32327, 11857, 58, 48753, 15591, 32848] avg=32768.0000 max=64288 max_rank=0 min=58 min_rank=4 imbalance_ratio=1.961914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[30498, 65460, 1371, 29036, 4507, 65526, 3797, 61949] avg=32768.0000 max=65526 max_rank=5 min=1371 min_rank=2 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[64288, 56422, 32327, 11857, 58, 48753, 15591, 32848] avg=32768.0000 max=64288 max_rank=0 min=58 min_rank=4 imbalance_ratio=1.961914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[64288, 56422, 32327, 11857, 58, 48753, 15591, 32848] avg=32768.0000 max=64288 max_rank=0 min=58 min_rank=4 imbalance_ratio=1.961914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[33095, 33095, 33097, 30472, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46967 max_load_reduction_pct=0.586618 moved_to_replicas=221374 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[33095, 33095, 33097, 30472, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46967 max_load_reduction_pct=0.586618 moved_to_replicas=221374 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[34363, 4988, 1145, 84121, 37197, 32171, 15662, 52497] avg=32768.0000 max=84121 max_rank=3 min=1145 min_rank=2 imbalance_ratio=2.567169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[33095, 33095, 33097, 30472, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46967 max_load_reduction_pct=0.586618 moved_to_replicas=221374 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[33096, 33095, 32327, 33097, 31489, 33096, 33096, 32848] avg=32768.0000 max=33097 max_rank=3 min=31489 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31191 max_load_reduction_pct=0.485176 moved_to_replicas=151459 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[30502, 33096, 33096, 33066, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30502 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=192212 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[33096, 33095, 32327, 33097, 31489, 33096, 33096, 32848] avg=32768.0000 max=33097 max_rank=3 min=31489 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31191 max_load_reduction_pct=0.485176 moved_to_replicas=151459 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[33096, 33095, 32327, 33097, 31489, 33096, 33096, 32848] avg=32768.0000 max=33097 max_rank=3 min=31489 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31191 max_load_reduction_pct=0.485176 moved_to_replicas=151459 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[33096, 31398, 33096, 33096, 33095, 32171, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31398 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=51024 max_load_reduction_pct=0.606555 moved_to_replicas=155671 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[64288, 56422, 32327, 11857, 58, 48753, 15591, 32848] avg=32768.0000 max=64288 max_rank=0 min=58 min_rank=4 imbalance_ratio=1.961914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[64288, 56422, 32327, 11857, 58, 48753, 15591, 32848] avg=32768.0000 max=64288 max_rank=0 min=58 min_rank=4 imbalance_ratio=1.961914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[64288, 56422, 32327, 11857, 58, 48753, 15591, 32848] avg=32768.0000 max=64288 max_rank=0 min=58 min_rank=4 imbalance_ratio=1.961914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[34363, 4988, 1145, 84121, 37197, 32171, 15662, 52497] avg=32768.0000 max=84121 max_rank=3 min=1145 min_rank=2 imbalance_ratio=2.567169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[34363, 4988, 1145, 84121, 37197, 32171, 15662, 52497] avg=32768.0000 max=84121 max_rank=3 min=1145 min_rank=2 imbalance_ratio=2.567169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[15033, 8682, 3252, 28195, 58170, 80064, 2524, 66224] avg=32768.0000 max=80064 max_rank=5 min=2524 min_rank=6 imbalance_ratio=2.443359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[34363, 4988, 1145, 84121, 37197, 32171, 15662, 52497] avg=32768.0000 max=84121 max_rank=3 min=1145 min_rank=2 imbalance_ratio=2.567169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[33096, 33095, 32327, 33097, 31489, 33096, 33096, 32848] avg=32768.0000 max=33097 max_rank=3 min=31489 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31191 max_load_reduction_pct=0.485176 moved_to_replicas=151459 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[33096, 33095, 32327, 33097, 31489, 33096, 33096, 32848] avg=32768.0000 max=33097 max_rank=3 min=31489 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31191 max_load_reduction_pct=0.485176 moved_to_replicas=151459 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[31525, 63245, 16166, 29705, 60697, 1338, 21039, 38429] avg=32768.0000 max=63245 max_rank=1 min=1338 min_rank=5 imbalance_ratio=1.930084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[33096, 33095, 32327, 33097, 31489, 33096, 33096, 32848] avg=32768.0000 max=33097 max_rank=3 min=31489 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31191 max_load_reduction_pct=0.485176 moved_to_replicas=151459 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[33096, 31398, 33096, 33096, 33095, 32171, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31398 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=51024 max_load_reduction_pct=0.606555 moved_to_replicas=155671 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[33096, 31398, 33096, 33096, 33095, 32171, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31398 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=51024 max_load_reduction_pct=0.606555 moved_to_replicas=155671 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[33095, 33095, 33097, 30472, 33097, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46967 max_load_reduction_pct=0.586618 moved_to_replicas=221374 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[33096, 31398, 33096, 33096, 33095, 32171, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31398 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=51024 max_load_reduction_pct=0.606555 moved_to_replicas=155671 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[31525, 33096, 33095, 33096, 33096, 33001, 32139, 33096] avg=32768.0000 max=33096 max_rank=1 min=31525 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30149 max_load_reduction_pct=0.476702 moved_to_replicas=128610 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[34363, 4988, 1145, 84121, 37197, 32171, 15662, 52497] avg=32768.0000 max=84121 max_rank=3 min=1145 min_rank=2 imbalance_ratio=2.567169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[34363, 4988, 1145, 84121, 37197, 32171, 15662, 52497] avg=32768.0000 max=84121 max_rank=3 min=1145 min_rank=2 imbalance_ratio=2.567169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[34363, 4988, 1145, 84121, 37197, 32171, 15662, 52497] avg=32768.0000 max=84121 max_rank=3 min=1145 min_rank=2 imbalance_ratio=2.567169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[31525, 63245, 16166, 29705, 60697, 1338, 21039, 38429] avg=32768.0000 max=63245 max_rank=1 min=1338 min_rank=5 imbalance_ratio=1.930084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[31525, 63245, 16166, 29705, 60697, 1338, 21039, 38429] avg=32768.0000 max=63245 max_rank=1 min=1338 min_rank=5 imbalance_ratio=1.930084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[31525, 63245, 16166, 29705, 60697, 1338, 21039, 38429] avg=32768.0000 max=63245 max_rank=1 min=1338 min_rank=5 imbalance_ratio=1.930084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[64288, 56422, 32327, 11857, 58, 48753, 15591, 32848] avg=32768.0000 max=64288 max_rank=0 min=58 min_rank=4 imbalance_ratio=1.961914 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[33096, 31398, 33096, 33096, 33095, 32171, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31398 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=51024 max_load_reduction_pct=0.606555 moved_to_replicas=155671 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[34967, 50795, 38257, 10339, 20590, 20209, 40791, 46196] avg=32768.0000 max=50795 max_rank=1 min=10339 min_rank=3 imbalance_ratio=1.550140 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[33096, 31398, 33096, 33096, 33095, 32171, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31398 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=51024 max_load_reduction_pct=0.606555 moved_to_replicas=155671 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[33096, 31398, 33096, 33096, 33095, 32171, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31398 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=51024 max_load_reduction_pct=0.606555 moved_to_replicas=155671 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[31525, 33096, 33095, 33096, 33096, 33001, 32139, 33096] avg=32768.0000 max=33096 max_rank=1 min=31525 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30149 max_load_reduction_pct=0.476702 moved_to_replicas=128610 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[31525, 33096, 33095, 33096, 33096, 33001, 32139, 33096] avg=32768.0000 max=33096 max_rank=1 min=31525 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30149 max_load_reduction_pct=0.476702 moved_to_replicas=128610 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[31525, 33096, 33095, 33096, 33096, 33001, 32139, 33096] avg=32768.0000 max=33096 max_rank=1 min=31525 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30149 max_load_reduction_pct=0.476702 moved_to_replicas=128610 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[33096, 33095, 32327, 33097, 31489, 33096, 33096, 32848] avg=32768.0000 max=33097 max_rank=3 min=31489 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31191 max_load_reduction_pct=0.485176 moved_to_replicas=151459 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=17699 max_load_reduction_pct=0.348440 moved_to_replicas=104585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[31525, 63245, 16166, 29705, 60697, 1338, 21039, 38429] avg=32768.0000 max=63245 max_rank=1 min=1338 min_rank=5 imbalance_ratio=1.930084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[31525, 63245, 16166, 29705, 60697, 1338, 21039, 38429] avg=32768.0000 max=63245 max_rank=1 min=1338 min_rank=5 imbalance_ratio=1.930084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[31525, 63245, 16166, 29705, 60697, 1338, 21039, 38429] avg=32768.0000 max=63245 max_rank=1 min=1338 min_rank=5 imbalance_ratio=1.930084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[34967, 50795, 38257, 10339, 20590, 20209, 40791, 46196] avg=32768.0000 max=50795 max_rank=1 min=10339 min_rank=3 imbalance_ratio=1.550140 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[34967, 50795, 38257, 10339, 20590, 20209, 40791, 46196] avg=32768.0000 max=50795 max_rank=1 min=10339 min_rank=3 imbalance_ratio=1.550140 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[34967, 50795, 38257, 10339, 20590, 20209, 40791, 46196] avg=32768.0000 max=50795 max_rank=1 min=10339 min_rank=3 imbalance_ratio=1.550140 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[367, 1787, 43489, 9069, 97570, 64808, 9275, 35779] avg=32768.0000 max=97570 max_rank=4 min=367 min_rank=0 imbalance_ratio=2.977600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[34363, 4988, 1145, 84121, 37197, 32171, 15662, 52497] avg=32768.0000 max=84121 max_rank=3 min=1145 min_rank=2 imbalance_ratio=2.567169 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[31525, 33096, 33095, 33096, 33096, 33001, 32139, 33096] avg=32768.0000 max=33096 max_rank=1 min=31525 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30149 max_load_reduction_pct=0.476702 moved_to_replicas=128610 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[31525, 33096, 33095, 33096, 33096, 33001, 32139, 33096] avg=32768.0000 max=33096 max_rank=1 min=31525 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30149 max_load_reduction_pct=0.476702 moved_to_replicas=128610 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[31525, 33096, 33095, 33096, 33096, 33001, 32139, 33096] avg=32768.0000 max=33096 max_rank=1 min=31525 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30149 max_load_reduction_pct=0.476702 moved_to_replicas=128610 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=17699 max_load_reduction_pct=0.348440 moved_to_replicas=104585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=17699 max_load_reduction_pct=0.348440 moved_to_replicas=104585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[32939, 33096, 33095, 30630, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30630 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64474 max_load_reduction_pct=0.660797 moved_to_replicas=228371 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=17699 max_load_reduction_pct=0.348440 moved_to_replicas=104585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[33096, 31398, 33096, 33096, 33095, 32171, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31398 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=51024 max_load_reduction_pct=0.606555 moved_to_replicas=155671 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[34967, 50795, 38257, 10339, 20590, 20209, 40791, 46196] avg=32768.0000 max=50795 max_rank=1 min=10339 min_rank=3 imbalance_ratio=1.550140 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[367, 1787, 43489, 9069, 97570, 64808, 9275, 35779] avg=32768.0000 max=97570 max_rank=4 min=367 min_rank=0 imbalance_ratio=2.977600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[34967, 50795, 38257, 10339, 20590, 20209, 40791, 46196] avg=32768.0000 max=50795 max_rank=1 min=10339 min_rank=3 imbalance_ratio=1.550140 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[367, 1787, 43489, 9069, 97570, 64808, 9275, 35779] avg=32768.0000 max=97570 max_rank=4 min=367 min_rank=0 imbalance_ratio=2.977600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[34967, 50795, 38257, 10339, 20590, 20209, 40791, 46196] avg=32768.0000 max=50795 max_rank=1 min=10339 min_rank=3 imbalance_ratio=1.550140 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[82151, 52384, 3350, 720, 9327, 41486, 32770, 39956] avg=32768.0000 max=82151 max_rank=0 min=720 min_rank=3 imbalance_ratio=2.507050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[367, 1787, 43489, 9069, 97570, 64808, 9275, 35779] avg=32768.0000 max=97570 max_rank=4 min=367 min_rank=0 imbalance_ratio=2.977600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[31525, 63245, 16166, 29705, 60697, 1338, 21039, 38429] avg=32768.0000 max=63245 max_rank=1 min=1338 min_rank=5 imbalance_ratio=1.930084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[32939, 33096, 33095, 30630, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30630 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64474 max_load_reduction_pct=0.660797 moved_to_replicas=228371 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=17699 max_load_reduction_pct=0.348440 moved_to_replicas=104585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=17699 max_load_reduction_pct=0.348440 moved_to_replicas=104585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[32939, 33096, 33095, 30630, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30630 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64474 max_load_reduction_pct=0.660797 moved_to_replicas=228371 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30798, 33095, 32770, 33096] avg=32768.0000 max=33097 max_rank=2 min=30798 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49054 max_load_reduction_pct=0.597120 moved_to_replicas=167532 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=17699 max_load_reduction_pct=0.348440 moved_to_replicas=104585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[32939, 33096, 33095, 30630, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30630 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64474 max_load_reduction_pct=0.660797 moved_to_replicas=228371 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[31525, 33096, 33095, 33096, 33096, 33001, 32139, 33096] avg=32768.0000 max=33096 max_rank=1 min=31525 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30149 max_load_reduction_pct=0.476702 moved_to_replicas=128610 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[82151, 52384, 3350, 720, 9327, 41486, 32770, 39956] avg=32768.0000 max=82151 max_rank=0 min=720 min_rank=3 imbalance_ratio=2.507050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[367, 1787, 43489, 9069, 97570, 64808, 9275, 35779] avg=32768.0000 max=97570 max_rank=4 min=367 min_rank=0 imbalance_ratio=2.977600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[367, 1787, 43489, 9069, 97570, 64808, 9275, 35779] avg=32768.0000 max=97570 max_rank=4 min=367 min_rank=0 imbalance_ratio=2.977600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[82151, 52384, 3350, 720, 9327, 41486, 32770, 39956] avg=32768.0000 max=82151 max_rank=0 min=720 min_rank=3 imbalance_ratio=2.507050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[39658, 57112, 30592, 1239, 32721, 10275, 31272, 59275] avg=32768.0000 max=59275 max_rank=7 min=1239 min_rank=3 imbalance_ratio=1.808929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[367, 1787, 43489, 9069, 97570, 64808, 9275, 35779] avg=32768.0000 max=97570 max_rank=4 min=367 min_rank=0 imbalance_ratio=2.977600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[82151, 52384, 3350, 720, 9327, 41486, 32770, 39956] avg=32768.0000 max=82151 max_rank=0 min=720 min_rank=3 imbalance_ratio=2.507050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30798, 33095, 32770, 33096] avg=32768.0000 max=33097 max_rank=2 min=30798 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49054 max_load_reduction_pct=0.597120 moved_to_replicas=167532 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[34967, 50795, 38257, 10339, 20590, 20209, 40791, 46196] avg=32768.0000 max=50795 max_rank=1 min=10339 min_rank=3 imbalance_ratio=1.550140 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[32939, 33096, 33095, 30630, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30630 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64474 max_load_reduction_pct=0.660797 moved_to_replicas=228371 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[32939, 33096, 33095, 30630, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30630 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64474 max_load_reduction_pct=0.660797 moved_to_replicas=228371 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30798, 33095, 32770, 33096] avg=32768.0000 max=33097 max_rank=2 min=30798 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49054 max_load_reduction_pct=0.597120 moved_to_replicas=167532 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[33095, 33097, 32671, 33098, 32721, 33095, 31272, 33095] avg=32768.0000 max=33098 max_rank=3 min=31272 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=26177 max_load_reduction_pct=0.441620 moved_to_replicas=116287 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[32939, 33096, 33095, 30630, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30630 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64474 max_load_reduction_pct=0.660797 moved_to_replicas=228371 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30798, 33095, 32770, 33096] avg=32768.0000 max=33097 max_rank=2 min=30798 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49054 max_load_reduction_pct=0.597120 moved_to_replicas=167532 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=17699 max_load_reduction_pct=0.348440 moved_to_replicas=104585 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[39658, 57112, 30592, 1239, 32721, 10275, 31272, 59275] avg=32768.0000 max=59275 max_rank=7 min=1239 min_rank=3 imbalance_ratio=1.808929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[82151, 52384, 3350, 720, 9327, 41486, 32770, 39956] avg=32768.0000 max=82151 max_rank=0 min=720 min_rank=3 imbalance_ratio=2.507050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[82151, 52384, 3350, 720, 9327, 41486, 32770, 39956] avg=32768.0000 max=82151 max_rank=0 min=720 min_rank=3 imbalance_ratio=2.507050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[39658, 57112, 30592, 1239, 32721, 10275, 31272, 59275] avg=32768.0000 max=59275 max_rank=7 min=1239 min_rank=3 imbalance_ratio=1.808929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[82151, 52384, 3350, 720, 9327, 41486, 32770, 39956] avg=32768.0000 max=82151 max_rank=0 min=720 min_rank=3 imbalance_ratio=2.507050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[39658, 57112, 30592, 1239, 32721, 10275, 31272, 59275] avg=32768.0000 max=59275 max_rank=7 min=1239 min_rank=3 imbalance_ratio=1.808929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[33095, 33097, 32671, 33098, 32721, 33095, 31272, 33095] avg=32768.0000 max=33098 max_rank=3 min=31272 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=26177 max_load_reduction_pct=0.441620 moved_to_replicas=116287 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30798, 33095, 32770, 33096] avg=32768.0000 max=33097 max_rank=2 min=30798 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49054 max_load_reduction_pct=0.597120 moved_to_replicas=167532 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30798, 33095, 32770, 33096] avg=32768.0000 max=33097 max_rank=2 min=30798 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49054 max_load_reduction_pct=0.597120 moved_to_replicas=167532 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[33095, 33097, 32671, 33098, 32721, 33095, 31272, 33095] avg=32768.0000 max=33098 max_rank=3 min=31272 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=26177 max_load_reduction_pct=0.441620 moved_to_replicas=116287 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[367, 1787, 43489, 9069, 97570, 64808, 9275, 35779] avg=32768.0000 max=97570 max_rank=4 min=367 min_rank=0 imbalance_ratio=2.977600 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30798, 33095, 32770, 33096] avg=32768.0000 max=33097 max_rank=2 min=30798 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49054 max_load_reduction_pct=0.597120 moved_to_replicas=167532 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[33095, 33097, 32671, 33098, 32721, 33095, 31272, 33095] avg=32768.0000 max=33098 max_rank=3 min=31272 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=26177 max_load_reduction_pct=0.441620 moved_to_replicas=116287 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[32939, 33096, 33095, 30630, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30630 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64474 max_load_reduction_pct=0.660797 moved_to_replicas=228371 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[39658, 57112, 30592, 1239, 32721, 10275, 31272, 59275] avg=32768.0000 max=59275 max_rank=7 min=1239 min_rank=3 imbalance_ratio=1.808929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[39658, 57112, 30592, 1239, 32721, 10275, 31272, 59275] avg=32768.0000 max=59275 max_rank=7 min=1239 min_rank=3 imbalance_ratio=1.808929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[39658, 57112, 30592, 1239, 32721, 10275, 31272, 59275] avg=32768.0000 max=59275 max_rank=7 min=1239 min_rank=3 imbalance_ratio=1.808929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[33095, 33097, 32671, 33098, 32721, 33095, 31272, 33095] avg=32768.0000 max=33098 max_rank=3 min=31272 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=26177 max_load_reduction_pct=0.441620 moved_to_replicas=116287 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[33095, 33097, 32671, 33098, 32721, 33095, 31272, 33095] avg=32768.0000 max=33098 max_rank=3 min=31272 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=26177 max_load_reduction_pct=0.441620 moved_to_replicas=116287 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[33095, 33097, 32671, 33098, 32721, 33095, 31272, 33095] avg=32768.0000 max=33098 max_rank=3 min=31272 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=26177 max_load_reduction_pct=0.441620 moved_to_replicas=116287 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[82151, 52384, 3350, 720, 9327, 41486, 32770, 39956] avg=32768.0000 max=82151 max_rank=0 min=720 min_rank=3 imbalance_ratio=2.507050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30798, 33095, 32770, 33096] avg=32768.0000 max=33097 max_rank=2 min=30798 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49054 max_load_reduction_pct=0.597120 moved_to_replicas=167532 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[39658, 57112, 30592, 1239, 32721, 10275, 31272, 59275] avg=32768.0000 max=59275 max_rank=7 min=1239 min_rank=3 imbalance_ratio=1.808929 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=32 total_expected_recv=1055796 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=47616 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[33095, 33097, 32671, 33098, 32721, 33095, 31272, 33095] avg=32768.0000 max=33098 max_rank=3 min=31272 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=26177 max_load_reduction_pct=0.441620 moved_to_replicas=116287 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=32 total_expected_recv=1040472 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=61952 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=32 total_expected_recv=1043243 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=59904 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=32 total_expected_recv=1049627 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=54400 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=32 total_expected_recv=1045037 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=58112 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=32 total_expected_recv=1057188 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=47872 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=32 total_expected_recv=1043857 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57984 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=32 total_expected_recv=1053388 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=49536 peak_demand=33097 required_capacity=1.010040 + [2026-08-30 05:11:41] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 571.7 | throughput per GPU (TFLOP/s/GPU): 583.1 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.028893E+01 | loss scale: 1.0 | grad norm: 201.560 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33097, 31781, 33096, 31787, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 31781, 33096, 31787, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=34816 expected_recv=64883 expected_overflow=0 exact_recv_per_rank=[33097, 31781, 33096, 31787, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=34816 expected_recv=64877 expected_overflow=0 exact_recv_per_rank=[33097, 31781, 33096, 31787, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 31781, 33096, 31787, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=34816 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[33097, 31781, 33096, 31787, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33097, 31781, 33096, 31787, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=34816 expected_recv=64883 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 31781, 33096, 31787, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=34816 expected_recv=64877 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=34816 expected_recv=63568 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=64512 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32023, 32292, 32776, 32669, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=34816 expected_recv=65658 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32023, 32292, 32776, 32669, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=34816 expected_recv=65388 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32023, 32292, 32776, 32669, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=34816 expected_recv=65658 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=34816 expected_recv=64941 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32023, 32292, 32776, 32669, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=34816 expected_recv=65388 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=34816 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32023, 32292, 32776, 32669, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32023, 32292, 32776, 32669, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=34816 expected_recv=64169 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32023, 32292, 32776, 32669, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=34816 expected_recv=65556 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32023, 32292, 32776, 32669, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=34816 expected_recv=64941 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=34816 expected_recv=66194 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=34816 expected_recv=64169 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=34816 expected_recv=65556 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=64896 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=34816 expected_recv=66105 expected_overflow=0 exact_recv_per_rank=[32842, 33096, 31853, 33096, 33096, 33050, 32311, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=34816 expected_recv=65730 expected_overflow=0 exact_recv_per_rank=[32842, 33096, 31853, 33096, 33096, 33050, 32311, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32842, 33096, 31853, 33096, 33096, 33050, 32311, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=34816 expected_recv=66105 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=34816 expected_recv=65730 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=34816 expected_recv=64188 expected_overflow=0 exact_recv_per_rank=[32842, 33096, 31853, 33096, 33096, 33050, 32311, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=67200 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=34816 expected_recv=64950 expected_overflow=0 exact_recv_per_rank=[32842, 33096, 31853, 33096, 33096, 33050, 32311, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=34816 expected_recv=64739 expected_overflow=0 exact_recv_per_rank=[32842, 33096, 31853, 33096, 33096, 33050, 32311, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32842, 33096, 31853, 33096, 33096, 33050, 32311, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=34816 expected_recv=64188 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32842, 33096, 31853, 33096, 33096, 33050, 32311, 32800] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=34816 expected_recv=64950 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=34816 expected_recv=64739 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31071, 32882, 33096, 33096, 33096, 33028, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31071, 32882, 33096, 33096, 33096, 33028, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31071, 32882, 33096, 33096, 33096, 33028, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=34816 expected_recv=65875 expected_overflow=0 exact_recv_per_rank=[33096, 31071, 32882, 33096, 33096, 33096, 33028, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=34816 expected_recv=65742 expected_overflow=0 exact_recv_per_rank=[33096, 31071, 32882, 33096, 33096, 33096, 33028, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=34816 expected_recv=65284 expected_overflow=0 exact_recv_per_rank=[33096, 31071, 32882, 33096, 33096, 33096, 33028, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=34816 expected_recv=65875 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=34816 expected_recv=64886 expected_overflow=0 exact_recv_per_rank=[33096, 31071, 32882, 33096, 33096, 33096, 33028, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=34816 expected_recv=63925 expected_overflow=0 exact_recv_per_rank=[33096, 31071, 32882, 33096, 33096, 33096, 33028, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=34816 expected_recv=65742 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=34816 expected_recv=65284 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=34816 expected_recv=64886 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=34816 expected_recv=65212 expected_overflow=0 exact_recv_per_rank=[31457, 33097, 33096, 32784, 33096, 32760, 32758, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=34816 expected_recv=63925 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=34816 expected_recv=63718 expected_overflow=0 exact_recv_per_rank=[31457, 33097, 33096, 32784, 33096, 32760, 32758, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=34816 expected_recv=65880 expected_overflow=0 exact_recv_per_rank=[31457, 33097, 33096, 32784, 33096, 32760, 32758, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=65024 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=34816 expected_recv=65212 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=34816 expected_recv=63718 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=34816 expected_recv=65880 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65024 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31457, 33097, 33096, 32784, 33096, 32760, 32758, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31457, 33097, 33096, 32784, 33096, 32760, 32758, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=34816 expected_recv=65456 expected_overflow=0 exact_recv_per_rank=[31457, 33097, 33096, 32784, 33096, 32760, 32758, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=34816 expected_recv=65445 expected_overflow=0 exact_recv_per_rank=[31457, 33097, 33096, 32784, 33096, 32760, 32758, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31457, 33097, 33096, 32784, 33096, 32760, 32758, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=34816 expected_recv=65456 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32795, 32727, 32118, 32365, 32851, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=34816 expected_recv=65445 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=34816 expected_recv=64792 expected_overflow=0 exact_recv_per_rank=[32795, 32727, 32118, 32365, 32851, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=34816 expected_recv=65685 expected_overflow=0 exact_recv_per_rank=[32795, 32727, 32118, 32365, 32851, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=34816 expected_recv=64792 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=34816 expected_recv=65685 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32795, 32727, 32118, 32365, 32851, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=34816 expected_recv=64605 expected_overflow=0 exact_recv_per_rank=[32795, 32727, 32118, 32365, 32851, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32795, 32727, 32118, 32365, 32851, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=34816 expected_recv=65276 expected_overflow=0 exact_recv_per_rank=[32795, 32727, 32118, 32365, 32851, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=34816 expected_recv=64605 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=34816 expected_recv=65354 expected_overflow=0 exact_recv_per_rank=[32795, 32727, 32118, 32365, 32851, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=34816 expected_recv=65276 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=34816 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33097, 32967, 32780, 32806, 33095, 33097, 31521, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=34816 expected_recv=65612 expected_overflow=0 exact_recv_per_rank=[33097, 32967, 32780, 32806, 33095, 33097, 31521, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33097, 32967, 32780, 32806, 33095, 33097, 31521, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=34816 expected_recv=65354 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=34816 expected_recv=66194 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=34816 expected_recv=65612 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=34816 expected_recv=65587 expected_overflow=0 exact_recv_per_rank=[33097, 32967, 32780, 32806, 33095, 33097, 31521, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=34816 expected_recv=65876 expected_overflow=0 exact_recv_per_rank=[33097, 32967, 32780, 32806, 33095, 33097, 31521, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=34816 expected_recv=62573 expected_overflow=0 exact_recv_per_rank=[33097, 32967, 32780, 32806, 33095, 33097, 31521, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=34816 expected_recv=65587 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=34816 expected_recv=65876 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=34816 expected_recv=62573 actual_valid_recv=63360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=34816 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[33097, 32967, 32780, 32806, 33095, 33097, 31521, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=63360 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=34816 expected_recv=66190 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=34816 expected_recv=66063 expected_overflow=0 exact_recv_per_rank=[33097, 32967, 32780, 32806, 33095, 33097, 31521, 32781] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=34816 expected_recv=64302 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32468, 33096, 31099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32468, 33096, 31099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=34816 expected_recv=65979 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32468, 33096, 31099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=34816 expected_recv=66063 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=34816 expected_recv=64302 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=34816 expected_recv=65979 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=34816 expected_recv=63046 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32468, 33096, 31099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32468, 33096, 31099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=34816 expected_recv=63046 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32468, 33096, 31099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=64256 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=34816 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32468, 33096, 31099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=34816 expected_recv=63959 expected_overflow=0 exact_recv_per_rank=[33096, 32776, 33096, 32781, 33096, 31900, 33096, 32303] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=34816 expected_recv=66194 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33097, 32468, 33096, 31099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32776, 33096, 32781, 33096, 31900, 33096, 32303] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=34816 expected_recv=65476 expected_overflow=0 exact_recv_per_rank=[33096, 32776, 33096, 32781, 33096, 31900, 33096, 32303] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=34816 expected_recv=63959 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=34816 expected_recv=65476 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=64640 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=34816 expected_recv=64986 expected_overflow=0 exact_recv_per_rank=[33096, 32776, 33096, 32781, 33096, 31900, 33096, 32303] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=34816 expected_recv=65788 expected_overflow=0 exact_recv_per_rank=[33096, 32776, 33096, 32781, 33096, 31900, 33096, 32303] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 32776, 33096, 32781, 33096, 31900, 33096, 32303] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=34816 expected_recv=64986 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=34816 expected_recv=65788 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32776, 33096, 32781, 33096, 31900, 33096, 32303] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=34816 expected_recv=65587 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32560, 32654, 32976, 32755, 32990, 32018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32560, 32654, 32976, 32755, 32990, 32018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=34816 expected_recv=65504 expected_overflow=0 exact_recv_per_rank=[33096, 32776, 33096, 32781, 33096, 31900, 33096, 32303] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=34816 expected_recv=65587 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=34816 expected_recv=65555 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32560, 32654, 32976, 32755, 32990, 32018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=34816 expected_recv=65504 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=34816 expected_recv=65555 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=34816 expected_recv=65406 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32560, 32654, 32976, 32755, 32990, 32018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=34816 expected_recv=63431 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32560, 32654, 32976, 32755, 32990, 32018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=34816 expected_recv=65406 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=34816 expected_recv=63431 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=34816 expected_recv=66086 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32560, 32654, 32976, 32755, 32990, 32018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=34816 expected_recv=65841 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32560, 32654, 32976, 32755, 32990, 32018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=64256 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=34816 expected_recv=66086 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=34816 expected_recv=65841 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=34816 expected_recv=65561 expected_overflow=0 exact_recv_per_rank=[32479, 32704, 33096, 32979, 31890, 32804, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=34816 expected_recv=64627 expected_overflow=0 exact_recv_per_rank=[32479, 32704, 33096, 32979, 31890, 32804, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=34816 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32560, 32654, 32976, 32755, 32990, 32018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=34816 expected_recv=65561 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=34816 expected_recv=66075 expected_overflow=0 exact_recv_per_rank=[32479, 32704, 33096, 32979, 31890, 32804, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=34816 expected_recv=64627 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=34816 expected_recv=66190 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=34816 expected_recv=66075 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32479, 32704, 33096, 32979, 31890, 32804, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32479, 32704, 33096, 32979, 31890, 32804, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=34816 expected_recv=63941 expected_overflow=0 exact_recv_per_rank=[32479, 32704, 33096, 32979, 31890, 32804, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32479, 32704, 33096, 32979, 31890, 32804, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=34816 expected_recv=63941 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=34816 expected_recv=65386 expected_overflow=0 exact_recv_per_rank=[32833, 32127, 33096, 32069, 33096, 32732, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=34816 expected_recv=65549 expected_overflow=0 exact_recv_per_rank=[32833, 32127, 33096, 32069, 33096, 32732, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=64896 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=34816 expected_recv=65508 expected_overflow=0 exact_recv_per_rank=[32479, 32704, 33096, 32979, 31890, 32804, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=34816 expected_recv=63672 expected_overflow=0 exact_recv_per_rank=[32833, 32127, 33096, 32069, 33096, 32732, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=34816 expected_recv=65386 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=34816 expected_recv=65549 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32833, 32127, 33096, 32069, 33096, 32732, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=34816 expected_recv=65508 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=34816 expected_recv=63672 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=64512 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32833, 32127, 33096, 32069, 33096, 32732, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32833, 32127, 33096, 32069, 33096, 32732, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32833, 32127, 33096, 32069, 33096, 32732, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=34816 expected_recv=63804 expected_overflow=0 exact_recv_per_rank=[33096, 32918, 32636, 32762, 33096, 32503, 32291, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32918, 32636, 32762, 33096, 32503, 32291, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=34816 expected_recv=65858 expected_overflow=0 exact_recv_per_rank=[33096, 32918, 32636, 32762, 33096, 32503, 32291, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=34816 expected_recv=64914 expected_overflow=0 exact_recv_per_rank=[32833, 32127, 33096, 32069, 33096, 32732, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=34816 expected_recv=63804 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=34816 expected_recv=65216 expected_overflow=0 exact_recv_per_rank=[33096, 32918, 32636, 32762, 33096, 32503, 32291, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=34816 expected_recv=65858 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=34816 expected_recv=64914 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=64512 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=34816 expected_recv=65216 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=34816 expected_recv=65938 expected_overflow=0 exact_recv_per_rank=[33096, 32918, 32636, 32762, 33096, 32503, 32291, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32918, 32636, 32762, 33096, 32503, 32291, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=34816 expected_recv=65938 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=34816 expected_recv=65387 expected_overflow=0 exact_recv_per_rank=[33096, 32918, 32636, 32762, 33096, 32503, 32291, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=34816 expected_recv=65866 expected_overflow=0 exact_recv_per_rank=[32720, 33096, 33096, 33095, 31921, 32770, 32350, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=34816 expected_recv=65387 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=34816 expected_recv=65816 expected_overflow=0 exact_recv_per_rank=[32720, 33096, 33096, 33095, 31921, 32770, 32350, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=34816 expected_recv=65866 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32720, 33096, 33096, 33095, 31921, 32770, 32350, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=34816 expected_recv=65701 expected_overflow=0 exact_recv_per_rank=[33096, 32918, 32636, 32762, 33096, 32503, 32291, 32842] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=34816 expected_recv=65816 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=34816 expected_recv=65398 expected_overflow=0 exact_recv_per_rank=[32720, 33096, 33096, 33095, 31921, 32770, 32350, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=34816 expected_recv=65701 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=34816 expected_recv=65398 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32720, 33096, 33096, 33095, 31921, 32770, 32350, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=34816 expected_recv=63444 expected_overflow=0 exact_recv_per_rank=[32720, 33096, 33096, 33095, 31921, 32770, 32350, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=34816 expected_recv=63444 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=34816 expected_recv=65189 expected_overflow=0 exact_recv_per_rank=[32720, 33096, 33096, 33095, 31921, 32770, 32350, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=64384 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=34816 expected_recv=65637 expected_overflow=0 exact_recv_per_rank=[32796, 33096, 33096, 32312, 33096, 32541, 32483, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=34816 expected_recv=65189 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=34816 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[32796, 33096, 33096, 32312, 33096, 32541, 32483, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=34816 expected_recv=65637 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=34816 expected_recv=64739 expected_overflow=0 exact_recv_per_rank=[32796, 33096, 33096, 32312, 33096, 32541, 32483, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32720, 33096, 33096, 33095, 31921, 32770, 32350, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=34816 expected_recv=65523 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32796, 33096, 33096, 32312, 33096, 32541, 32483, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=34816 expected_recv=64739 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=34816 expected_recv=64908 expected_overflow=0 exact_recv_per_rank=[32796, 33096, 33096, 32312, 33096, 32541, 32483, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32796, 33096, 33096, 32312, 33096, 32541, 32483, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=34816 expected_recv=64908 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=34816 expected_recv=64905 expected_overflow=0 exact_recv_per_rank=[32796, 33096, 33096, 32312, 33096, 32541, 32483, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31382, 32705, 33096, 33096, 32763, 33096, 32910, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=34816 expected_recv=62975 expected_overflow=0 exact_recv_per_rank=[31382, 32705, 33096, 33096, 32763, 33096, 32910, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=34816 expected_recv=64905 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31382, 32705, 33096, 33096, 32763, 33096, 32910, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=34816 expected_recv=62975 actual_valid_recv=64128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31382, 32705, 33096, 33096, 32763, 33096, 32910, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32796, 33096, 33096, 32312, 33096, 32541, 32483, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=64128 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[31382, 32705, 33096, 33096, 32763, 33096, 32910, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=34816 expected_recv=65615 expected_overflow=0 exact_recv_per_rank=[31382, 32705, 33096, 33096, 32763, 33096, 32910, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=34816 expected_recv=65615 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=34816 expected_recv=66006 expected_overflow=0 exact_recv_per_rank=[31382, 32705, 33096, 33096, 32763, 33096, 32910, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[40715, 30489, 44915, 10905, 53978, 41873, 20794, 18475] avg=32768.0000 max=53978 max_rank=4 min=10905 min_rank=3 imbalance_ratio=1.647278 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=34816 expected_recv=66006 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[40715, 30489, 44915, 10905, 53978, 41873, 20794, 18475] avg=32768.0000 max=53978 max_rank=4 min=10905 min_rank=3 imbalance_ratio=1.647278 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[40715, 30489, 44915, 10905, 53978, 41873, 20794, 18475] avg=32768.0000 max=53978 max_rank=4 min=10905 min_rank=3 imbalance_ratio=1.647278 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=34816 expected_recv=64924 expected_overflow=0 exact_recv_per_rank=[31382, 32705, 33096, 33096, 32763, 33096, 32910, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[40715, 30489, 44915, 10905, 53978, 41873, 20794, 18475] avg=32768.0000 max=53978 max_rank=4 min=10905 min_rank=3 imbalance_ratio=1.647278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[33097, 31781, 33096, 31787, 33096, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31781 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20881 max_load_reduction_pct=0.386843 moved_to_replicas=98372 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=34816 expected_recv=64924 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[40715, 30489, 44915, 10905, 53978, 41873, 20794, 18475] avg=32768.0000 max=53978 max_rank=4 min=10905 min_rank=3 imbalance_ratio=1.647278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[40715, 30489, 44915, 10905, 53978, 41873, 20794, 18475] avg=32768.0000 max=53978 max_rank=4 min=10905 min_rank=3 imbalance_ratio=1.647278 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[33097, 31781, 33096, 31787, 33096, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31781 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20881 max_load_reduction_pct=0.386843 moved_to_replicas=98372 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[33097, 31781, 33096, 31787, 33096, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31781 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20881 max_load_reduction_pct=0.386843 moved_to_replicas=98372 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[33097, 31781, 33096, 31787, 33096, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31781 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20881 max_load_reduction_pct=0.386843 moved_to_replicas=98372 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[40715, 30489, 44915, 10905, 53978, 41873, 20794, 18475] avg=32768.0000 max=53978 max_rank=4 min=10905 min_rank=3 imbalance_ratio=1.647278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[33097, 31781, 33096, 31787, 33096, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31781 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20881 max_load_reduction_pct=0.386843 moved_to_replicas=98372 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[33097, 31781, 33096, 31787, 33096, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31781 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20881 max_load_reduction_pct=0.386843 moved_to_replicas=98372 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[41851, 743, 32023, 31501, 30, 32669, 93496, 29831] avg=32768.0000 max=93496 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.853271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[41851, 743, 32023, 31501, 30, 32669, 93496, 29831] avg=32768.0000 max=93496 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.853271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[40715, 30489, 44915, 10905, 53978, 41873, 20794, 18475] avg=32768.0000 max=53978 max_rank=4 min=10905 min_rank=3 imbalance_ratio=1.647278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[41851, 743, 32023, 31501, 30, 32669, 93496, 29831] avg=32768.0000 max=93496 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.853271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[41851, 743, 32023, 31501, 30, 32669, 93496, 29831] avg=32768.0000 max=93496 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.853271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[33097, 31781, 33096, 31787, 33096, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31781 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20881 max_load_reduction_pct=0.386843 moved_to_replicas=98372 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33096, 32023, 32292, 32776, 32669, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32023 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=60400 max_load_reduction_pct=0.646017 moved_to_replicas=138052 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[41851, 743, 32023, 31501, 30, 32669, 93496, 29831] avg=32768.0000 max=93496 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.853271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[41851, 743, 32023, 31501, 30, 32669, 93496, 29831] avg=32768.0000 max=93496 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.853271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33096, 32023, 32292, 32776, 32669, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32023 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=60400 max_load_reduction_pct=0.646017 moved_to_replicas=138052 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33096, 32023, 32292, 32776, 32669, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32023 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=60400 max_load_reduction_pct=0.646017 moved_to_replicas=138052 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[33097, 31781, 33096, 31787, 33096, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31781 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=20881 max_load_reduction_pct=0.386843 moved_to_replicas=98372 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33096, 32023, 32292, 32776, 32669, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32023 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=60400 max_load_reduction_pct=0.646017 moved_to_replicas=138052 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33096, 32023, 32292, 32776, 32669, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32023 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=60400 max_load_reduction_pct=0.646017 moved_to_replicas=138052 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[123, 65146, 29933, 98507, 35016, 958, 32311, 150] avg=32768.0000 max=98507 max_rank=3 min=123 min_rank=0 imbalance_ratio=3.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33096, 32023, 32292, 32776, 32669, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32023 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=60400 max_load_reduction_pct=0.646017 moved_to_replicas=138052 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[41851, 743, 32023, 31501, 30, 32669, 93496, 29831] avg=32768.0000 max=93496 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.853271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[123, 65146, 29933, 98507, 35016, 958, 32311, 150] avg=32768.0000 max=98507 max_rank=3 min=123 min_rank=0 imbalance_ratio=3.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[123, 65146, 29933, 98507, 35016, 958, 32311, 150] avg=32768.0000 max=98507 max_rank=3 min=123 min_rank=0 imbalance_ratio=3.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[123, 65146, 29933, 98507, 35016, 958, 32311, 150] avg=32768.0000 max=98507 max_rank=3 min=123 min_rank=0 imbalance_ratio=3.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[41851, 743, 32023, 31501, 30, 32669, 93496, 29831] avg=32768.0000 max=93496 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.853271 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[32842, 33096, 31853, 33096, 33096, 33050, 32311, 32800] avg=32768.0000 max=33096 max_rank=1 min=31853 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65411 max_load_reduction_pct=0.664024 moved_to_replicas=199360 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33096, 32023, 32292, 32776, 32669, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32023 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=60400 max_load_reduction_pct=0.646017 moved_to_replicas=138052 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[123, 65146, 29933, 98507, 35016, 958, 32311, 150] avg=32768.0000 max=98507 max_rank=3 min=123 min_rank=0 imbalance_ratio=3.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[123, 65146, 29933, 98507, 35016, 958, 32311, 150] avg=32768.0000 max=98507 max_rank=3 min=123 min_rank=0 imbalance_ratio=3.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[32842, 33096, 31853, 33096, 33096, 33050, 32311, 32800] avg=32768.0000 max=33096 max_rank=1 min=31853 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65411 max_load_reduction_pct=0.664024 moved_to_replicas=199360 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[32842, 33096, 31853, 33096, 33096, 33050, 32311, 32800] avg=32768.0000 max=33096 max_rank=1 min=31853 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65411 max_load_reduction_pct=0.664024 moved_to_replicas=199360 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[32842, 33096, 31853, 33096, 33096, 33050, 32311, 32800] avg=32768.0000 max=33096 max_rank=1 min=31853 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65411 max_load_reduction_pct=0.664024 moved_to_replicas=199360 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33096, 32023, 32292, 32776, 32669, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32023 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=60400 max_load_reduction_pct=0.646017 moved_to_replicas=138052 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[32842, 33096, 31853, 33096, 33096, 33050, 32311, 32800] avg=32768.0000 max=33096 max_rank=1 min=31853 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65411 max_load_reduction_pct=0.664024 moved_to_replicas=199360 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[33721, 1231, 32882, 94490, 2260, 64457, 324, 32779] avg=32768.0000 max=94490 max_rank=3 min=324 min_rank=6 imbalance_ratio=2.883606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[32842, 33096, 31853, 33096, 33096, 33050, 32311, 32800] avg=32768.0000 max=33096 max_rank=1 min=31853 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65411 max_load_reduction_pct=0.664024 moved_to_replicas=199360 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[123, 65146, 29933, 98507, 35016, 958, 32311, 150] avg=32768.0000 max=98507 max_rank=3 min=123 min_rank=0 imbalance_ratio=3.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[33721, 1231, 32882, 94490, 2260, 64457, 324, 32779] avg=32768.0000 max=94490 max_rank=3 min=324 min_rank=6 imbalance_ratio=2.883606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[33721, 1231, 32882, 94490, 2260, 64457, 324, 32779] avg=32768.0000 max=94490 max_rank=3 min=324 min_rank=6 imbalance_ratio=2.883606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[33721, 1231, 32882, 94490, 2260, 64457, 324, 32779] avg=32768.0000 max=94490 max_rank=3 min=324 min_rank=6 imbalance_ratio=2.883606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[33096, 31071, 32882, 33096, 33096, 33096, 33028, 32779] avg=32768.0000 max=33096 max_rank=0 min=31071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61394 max_load_reduction_pct=0.649741 moved_to_replicas=189479 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[123, 65146, 29933, 98507, 35016, 958, 32311, 150] avg=32768.0000 max=98507 max_rank=3 min=123 min_rank=0 imbalance_ratio=3.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[32842, 33096, 31853, 33096, 33096, 33050, 32311, 32800] avg=32768.0000 max=33096 max_rank=1 min=31853 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65411 max_load_reduction_pct=0.664024 moved_to_replicas=199360 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[33721, 1231, 32882, 94490, 2260, 64457, 324, 32779] avg=32768.0000 max=94490 max_rank=3 min=324 min_rank=6 imbalance_ratio=2.883606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[33721, 1231, 32882, 94490, 2260, 64457, 324, 32779] avg=32768.0000 max=94490 max_rank=3 min=324 min_rank=6 imbalance_ratio=2.883606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[33096, 31071, 32882, 33096, 33096, 33096, 33028, 32779] avg=32768.0000 max=33096 max_rank=0 min=31071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61394 max_load_reduction_pct=0.649741 moved_to_replicas=189479 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[33096, 31071, 32882, 33096, 33096, 33096, 33028, 32779] avg=32768.0000 max=33096 max_rank=0 min=31071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61394 max_load_reduction_pct=0.649741 moved_to_replicas=189479 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[33096, 31071, 32882, 33096, 33096, 33096, 33028, 32779] avg=32768.0000 max=33096 max_rank=0 min=31071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61394 max_load_reduction_pct=0.649741 moved_to_replicas=189479 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[32842, 33096, 31853, 33096, 33096, 33050, 32311, 32800] avg=32768.0000 max=33096 max_rank=1 min=31853 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65411 max_load_reduction_pct=0.664024 moved_to_replicas=199360 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[33096, 31071, 32882, 33096, 33096, 33096, 33028, 32779] avg=32768.0000 max=33096 max_rank=0 min=31071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61394 max_load_reduction_pct=0.649741 moved_to_replicas=189479 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[143, 38478, 37825, 32784, 29196, 32760, 32758, 58200] avg=32768.0000 max=58200 max_rank=7 min=143 min_rank=0 imbalance_ratio=1.776123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[33096, 31071, 32882, 33096, 33096, 33096, 33028, 32779] avg=32768.0000 max=33096 max_rank=0 min=31071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61394 max_load_reduction_pct=0.649741 moved_to_replicas=189479 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[33721, 1231, 32882, 94490, 2260, 64457, 324, 32779] avg=32768.0000 max=94490 max_rank=3 min=324 min_rank=6 imbalance_ratio=2.883606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[143, 38478, 37825, 32784, 29196, 32760, 32758, 58200] avg=32768.0000 max=58200 max_rank=7 min=143 min_rank=0 imbalance_ratio=1.776123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[31457, 33097, 33096, 32784, 33096, 32760, 32758, 33096] avg=32768.0000 max=33097 max_rank=1 min=31457 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=25103 max_load_reduction_pct=0.431323 moved_to_replicas=67066 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[143, 38478, 37825, 32784, 29196, 32760, 32758, 58200] avg=32768.0000 max=58200 max_rank=7 min=143 min_rank=0 imbalance_ratio=1.776123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[143, 38478, 37825, 32784, 29196, 32760, 32758, 58200] avg=32768.0000 max=58200 max_rank=7 min=143 min_rank=0 imbalance_ratio=1.776123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[33721, 1231, 32882, 94490, 2260, 64457, 324, 32779] avg=32768.0000 max=94490 max_rank=3 min=324 min_rank=6 imbalance_ratio=2.883606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[143, 38478, 37825, 32784, 29196, 32760, 32758, 58200] avg=32768.0000 max=58200 max_rank=7 min=143 min_rank=0 imbalance_ratio=1.776123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[33096, 31071, 32882, 33096, 33096, 33096, 33028, 32779] avg=32768.0000 max=33096 max_rank=0 min=31071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61394 max_load_reduction_pct=0.649741 moved_to_replicas=189479 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[143, 38478, 37825, 32784, 29196, 32760, 32758, 58200] avg=32768.0000 max=58200 max_rank=7 min=143 min_rank=0 imbalance_ratio=1.776123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[31457, 33097, 33096, 32784, 33096, 32760, 32758, 33096] avg=32768.0000 max=33097 max_rank=1 min=31457 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=25103 max_load_reduction_pct=0.431323 moved_to_replicas=67066 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[31457, 33097, 33096, 32784, 33096, 32760, 32758, 33096] avg=32768.0000 max=33097 max_rank=1 min=31457 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=25103 max_load_reduction_pct=0.431323 moved_to_replicas=67066 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[31457, 33097, 33096, 32784, 33096, 32760, 32758, 33096] avg=32768.0000 max=33097 max_rank=1 min=31457 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=25103 max_load_reduction_pct=0.431323 moved_to_replicas=67066 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[33096, 31071, 32882, 33096, 33096, 33096, 33028, 32779] avg=32768.0000 max=33096 max_rank=0 min=31071 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61394 max_load_reduction_pct=0.649741 moved_to_replicas=189479 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[31457, 33097, 33096, 32784, 33096, 32760, 32758, 33096] avg=32768.0000 max=33097 max_rank=1 min=31457 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=25103 max_load_reduction_pct=0.431323 moved_to_replicas=67066 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[32795, 32727, 914, 106, 32851, 33778, 65355, 63618] avg=32768.0000 max=65355 max_rank=6 min=106 min_rank=3 imbalance_ratio=1.994476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[31457, 33097, 33096, 32784, 33096, 32760, 32758, 33096] avg=32768.0000 max=33097 max_rank=1 min=31457 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=25103 max_load_reduction_pct=0.431323 moved_to_replicas=67066 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[143, 38478, 37825, 32784, 29196, 32760, 32758, 58200] avg=32768.0000 max=58200 max_rank=7 min=143 min_rank=0 imbalance_ratio=1.776123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[32795, 32727, 914, 106, 32851, 33778, 65355, 63618] avg=32768.0000 max=65355 max_rank=6 min=106 min_rank=3 imbalance_ratio=1.994476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[32795, 32727, 32118, 32365, 32851, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32118 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32259 max_load_reduction_pct=0.493597 moved_to_replicas=127571 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[32795, 32727, 914, 106, 32851, 33778, 65355, 63618] avg=32768.0000 max=65355 max_rank=6 min=106 min_rank=3 imbalance_ratio=1.994476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[32795, 32727, 914, 106, 32851, 33778, 65355, 63618] avg=32768.0000 max=65355 max_rank=6 min=106 min_rank=3 imbalance_ratio=1.994476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[143, 38478, 37825, 32784, 29196, 32760, 32758, 58200] avg=32768.0000 max=58200 max_rank=7 min=143 min_rank=0 imbalance_ratio=1.776123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[32795, 32727, 914, 106, 32851, 33778, 65355, 63618] avg=32768.0000 max=65355 max_rank=6 min=106 min_rank=3 imbalance_ratio=1.994476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[31457, 33097, 33096, 32784, 33096, 32760, 32758, 33096] avg=32768.0000 max=33097 max_rank=1 min=31457 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=25103 max_load_reduction_pct=0.431323 moved_to_replicas=67066 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[32795, 32727, 32118, 32365, 32851, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32118 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32259 max_load_reduction_pct=0.493597 moved_to_replicas=127571 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[32795, 32727, 914, 106, 32851, 33778, 65355, 63618] avg=32768.0000 max=65355 max_rank=6 min=106 min_rank=3 imbalance_ratio=1.994476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[32795, 32727, 32118, 32365, 32851, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32118 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32259 max_load_reduction_pct=0.493597 moved_to_replicas=127571 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[32795, 32727, 32118, 32365, 32851, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32118 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32259 max_load_reduction_pct=0.493597 moved_to_replicas=127571 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[31457, 33097, 33096, 32784, 33096, 32760, 32758, 33096] avg=32768.0000 max=33097 max_rank=1 min=31457 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=25103 max_load_reduction_pct=0.431323 moved_to_replicas=67066 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[32795, 32727, 32118, 32365, 32851, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32118 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32259 max_load_reduction_pct=0.493597 moved_to_replicas=127571 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[10753, 32967, 32780, 32806, 5366, 90385, 24306, 32781] avg=32768.0000 max=90385 max_rank=5 min=5366 min_rank=4 imbalance_ratio=2.758331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[32795, 32727, 32118, 32365, 32851, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32118 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32259 max_load_reduction_pct=0.493597 moved_to_replicas=127571 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[32795, 32727, 914, 106, 32851, 33778, 65355, 63618] avg=32768.0000 max=65355 max_rank=6 min=106 min_rank=3 imbalance_ratio=1.994476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[10753, 32967, 32780, 32806, 5366, 90385, 24306, 32781] avg=32768.0000 max=90385 max_rank=5 min=5366 min_rank=4 imbalance_ratio=2.758331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[10753, 32967, 32780, 32806, 5366, 90385, 24306, 32781] avg=32768.0000 max=90385 max_rank=5 min=5366 min_rank=4 imbalance_ratio=2.758331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[10753, 32967, 32780, 32806, 5366, 90385, 24306, 32781] avg=32768.0000 max=90385 max_rank=5 min=5366 min_rank=4 imbalance_ratio=2.758331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[32795, 32727, 914, 106, 32851, 33778, 65355, 63618] avg=32768.0000 max=65355 max_rank=6 min=106 min_rank=3 imbalance_ratio=1.994476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32967, 32780, 32806, 33095, 33097, 31521, 32781] avg=32768.0000 max=33097 max_rank=0 min=31521 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=57288 max_load_reduction_pct=0.633822 moved_to_replicas=117508 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[10753, 32967, 32780, 32806, 5366, 90385, 24306, 32781] avg=32768.0000 max=90385 max_rank=5 min=5366 min_rank=4 imbalance_ratio=2.758331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[32795, 32727, 32118, 32365, 32851, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32118 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32259 max_load_reduction_pct=0.493597 moved_to_replicas=127571 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32967, 32780, 32806, 33095, 33097, 31521, 32781] avg=32768.0000 max=33097 max_rank=0 min=31521 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=57288 max_load_reduction_pct=0.633822 moved_to_replicas=117508 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[10753, 32967, 32780, 32806, 5366, 90385, 24306, 32781] avg=32768.0000 max=90385 max_rank=5 min=5366 min_rank=4 imbalance_ratio=2.758331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32967, 32780, 32806, 33095, 33097, 31521, 32781] avg=32768.0000 max=33097 max_rank=0 min=31521 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=57288 max_load_reduction_pct=0.633822 moved_to_replicas=117508 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32967, 32780, 32806, 33095, 33097, 31521, 32781] avg=32768.0000 max=33097 max_rank=0 min=31521 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=57288 max_load_reduction_pct=0.633822 moved_to_replicas=117508 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[32795, 32727, 32118, 32365, 32851, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32118 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32259 max_load_reduction_pct=0.493597 moved_to_replicas=127571 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32967, 32780, 32806, 33095, 33097, 31521, 32781] avg=32768.0000 max=33097 max_rank=0 min=31521 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=57288 max_load_reduction_pct=0.633822 moved_to_replicas=117508 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[62831, 3630, 38639, 33255, 54256, 836, 64728, 3969] avg=32768.0000 max=64728 max_rank=6 min=836 min_rank=5 imbalance_ratio=1.975342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32967, 32780, 32806, 33095, 33097, 31521, 32781] avg=32768.0000 max=33097 max_rank=0 min=31521 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=57288 max_load_reduction_pct=0.633822 moved_to_replicas=117508 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[10753, 32967, 32780, 32806, 5366, 90385, 24306, 32781] avg=32768.0000 max=90385 max_rank=5 min=5366 min_rank=4 imbalance_ratio=2.758331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[62831, 3630, 38639, 33255, 54256, 836, 64728, 3969] avg=32768.0000 max=64728 max_rank=6 min=836 min_rank=5 imbalance_ratio=1.975342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[62831, 3630, 38639, 33255, 54256, 836, 64728, 3969] avg=32768.0000 max=64728 max_rank=6 min=836 min_rank=5 imbalance_ratio=1.975342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32468, 33096, 31099] avg=32768.0000 max=33097 max_rank=4 min=31099 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31631 max_load_reduction_pct=0.488676 moved_to_replicas=169946 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[10753, 32967, 32780, 32806, 5366, 90385, 24306, 32781] avg=32768.0000 max=90385 max_rank=5 min=5366 min_rank=4 imbalance_ratio=2.758331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[62831, 3630, 38639, 33255, 54256, 836, 64728, 3969] avg=32768.0000 max=64728 max_rank=6 min=836 min_rank=5 imbalance_ratio=1.975342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[62831, 3630, 38639, 33255, 54256, 836, 64728, 3969] avg=32768.0000 max=64728 max_rank=6 min=836 min_rank=5 imbalance_ratio=1.975342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32967, 32780, 32806, 33095, 33097, 31521, 32781] avg=32768.0000 max=33097 max_rank=0 min=31521 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=57288 max_load_reduction_pct=0.633822 moved_to_replicas=117508 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32468, 33096, 31099] avg=32768.0000 max=33097 max_rank=4 min=31099 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31631 max_load_reduction_pct=0.488676 moved_to_replicas=169946 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[62831, 3630, 38639, 33255, 54256, 836, 64728, 3969] avg=32768.0000 max=64728 max_rank=6 min=836 min_rank=5 imbalance_ratio=1.975342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32468, 33096, 31099] avg=32768.0000 max=33097 max_rank=4 min=31099 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31631 max_load_reduction_pct=0.488676 moved_to_replicas=169946 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[33097, 32967, 32780, 32806, 33095, 33097, 31521, 32781] avg=32768.0000 max=33097 max_rank=0 min=31521 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=57288 max_load_reduction_pct=0.633822 moved_to_replicas=117508 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32468, 33096, 31099] avg=32768.0000 max=33097 max_rank=4 min=31099 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31631 max_load_reduction_pct=0.488676 moved_to_replicas=169946 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32468, 33096, 31099] avg=32768.0000 max=33097 max_rank=4 min=31099 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31631 max_load_reduction_pct=0.488676 moved_to_replicas=169946 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[66194, 50, 33452, 32781, 65451, 26032, 5881, 32303] avg=32768.0000 max=66194 max_rank=0 min=50 min_rank=1 imbalance_ratio=2.020081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32468, 33096, 31099] avg=32768.0000 max=33097 max_rank=4 min=31099 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31631 max_load_reduction_pct=0.488676 moved_to_replicas=169946 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[62831, 3630, 38639, 33255, 54256, 836, 64728, 3969] avg=32768.0000 max=64728 max_rank=6 min=836 min_rank=5 imbalance_ratio=1.975342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[66194, 50, 33452, 32781, 65451, 26032, 5881, 32303] avg=32768.0000 max=66194 max_rank=0 min=50 min_rank=1 imbalance_ratio=2.020081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32776, 33096, 32781, 33096, 31900, 33096, 32303] avg=32768.0000 max=33096 max_rank=0 min=31900 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33098 max_load_reduction_pct=0.500015 moved_to_replicas=131750 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[66194, 50, 33452, 32781, 65451, 26032, 5881, 32303] avg=32768.0000 max=66194 max_rank=0 min=50 min_rank=1 imbalance_ratio=2.020081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[62831, 3630, 38639, 33255, 54256, 836, 64728, 3969] avg=32768.0000 max=64728 max_rank=6 min=836 min_rank=5 imbalance_ratio=1.975342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[66194, 50, 33452, 32781, 65451, 26032, 5881, 32303] avg=32768.0000 max=66194 max_rank=0 min=50 min_rank=1 imbalance_ratio=2.020081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[66194, 50, 33452, 32781, 65451, 26032, 5881, 32303] avg=32768.0000 max=66194 max_rank=0 min=50 min_rank=1 imbalance_ratio=2.020081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32468, 33096, 31099] avg=32768.0000 max=33097 max_rank=4 min=31099 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31631 max_load_reduction_pct=0.488676 moved_to_replicas=169946 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32776, 33096, 32781, 33096, 31900, 33096, 32303] avg=32768.0000 max=33096 max_rank=0 min=31900 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33098 max_load_reduction_pct=0.500015 moved_to_replicas=131750 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[66194, 50, 33452, 32781, 65451, 26032, 5881, 32303] avg=32768.0000 max=66194 max_rank=0 min=50 min_rank=1 imbalance_ratio=2.020081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32776, 33096, 32781, 33096, 31900, 33096, 32303] avg=32768.0000 max=33096 max_rank=0 min=31900 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33098 max_load_reduction_pct=0.500015 moved_to_replicas=131750 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33097, 32468, 33096, 31099] avg=32768.0000 max=33097 max_rank=4 min=31099 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=31631 max_load_reduction_pct=0.488676 moved_to_replicas=169946 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32776, 33096, 32781, 33096, 31900, 33096, 32303] avg=32768.0000 max=33096 max_rank=0 min=31900 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33098 max_load_reduction_pct=0.500015 moved_to_replicas=131750 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[97799, 7291, 32560, 32654, 32976, 44, 32990, 25830] avg=32768.0000 max=97799 max_rank=0 min=44 min_rank=5 imbalance_ratio=2.984589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32776, 33096, 32781, 33096, 31900, 33096, 32303] avg=32768.0000 max=33096 max_rank=0 min=31900 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33098 max_load_reduction_pct=0.500015 moved_to_replicas=131750 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32776, 33096, 32781, 33096, 31900, 33096, 32303] avg=32768.0000 max=33096 max_rank=0 min=31900 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33098 max_load_reduction_pct=0.500015 moved_to_replicas=131750 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[97799, 7291, 32560, 32654, 32976, 44, 32990, 25830] avg=32768.0000 max=97799 max_rank=0 min=44 min_rank=5 imbalance_ratio=2.984589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[66194, 50, 33452, 32781, 65451, 26032, 5881, 32303] avg=32768.0000 max=66194 max_rank=0 min=50 min_rank=1 imbalance_ratio=2.020081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[33096, 33095, 32560, 32654, 32976, 32755, 32990, 32018] avg=32768.0000 max=33096 max_rank=0 min=32018 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64703 max_load_reduction_pct=0.661592 moved_to_replicas=128390 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[97799, 7291, 32560, 32654, 32976, 44, 32990, 25830] avg=32768.0000 max=97799 max_rank=0 min=44 min_rank=5 imbalance_ratio=2.984589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[66194, 50, 33452, 32781, 65451, 26032, 5881, 32303] avg=32768.0000 max=66194 max_rank=0 min=50 min_rank=1 imbalance_ratio=2.020081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[97799, 7291, 32560, 32654, 32976, 44, 32990, 25830] avg=32768.0000 max=97799 max_rank=0 min=44 min_rank=5 imbalance_ratio=2.984589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[33096, 33095, 32560, 32654, 32976, 32755, 32990, 32018] avg=32768.0000 max=33096 max_rank=0 min=32018 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64703 max_load_reduction_pct=0.661592 moved_to_replicas=128390 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[97799, 7291, 32560, 32654, 32976, 44, 32990, 25830] avg=32768.0000 max=97799 max_rank=0 min=44 min_rank=5 imbalance_ratio=2.984589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32776, 33096, 32781, 33096, 31900, 33096, 32303] avg=32768.0000 max=33096 max_rank=0 min=31900 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33098 max_load_reduction_pct=0.500015 moved_to_replicas=131750 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[33096, 33095, 32560, 32654, 32976, 32755, 32990, 32018] avg=32768.0000 max=33096 max_rank=0 min=32018 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64703 max_load_reduction_pct=0.661592 moved_to_replicas=128390 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32776, 33096, 32781, 33096, 31900, 33096, 32303] avg=32768.0000 max=33096 max_rank=0 min=31900 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33098 max_load_reduction_pct=0.500015 moved_to_replicas=131750 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[33096, 33095, 32560, 32654, 32976, 32755, 32990, 32018] avg=32768.0000 max=33096 max_rank=0 min=32018 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64703 max_load_reduction_pct=0.661592 moved_to_replicas=128390 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[134, 32704, 65251, 824, 26940, 32804, 38046, 65441] avg=32768.0000 max=65441 max_rank=7 min=134 min_rank=0 imbalance_ratio=1.997101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[33096, 33095, 32560, 32654, 32976, 32755, 32990, 32018] avg=32768.0000 max=33096 max_rank=0 min=32018 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64703 max_load_reduction_pct=0.661592 moved_to_replicas=128390 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[97799, 7291, 32560, 32654, 32976, 44, 32990, 25830] avg=32768.0000 max=97799 max_rank=0 min=44 min_rank=5 imbalance_ratio=2.984589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[134, 32704, 65251, 824, 26940, 32804, 38046, 65441] avg=32768.0000 max=65441 max_rank=7 min=134 min_rank=0 imbalance_ratio=1.997101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[97799, 7291, 32560, 32654, 32976, 44, 32990, 25830] avg=32768.0000 max=97799 max_rank=0 min=44 min_rank=5 imbalance_ratio=2.984589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[32479, 32704, 33096, 32979, 31890, 32804, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31890 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32345 max_load_reduction_pct=0.494262 moved_to_replicas=134657 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[134, 32704, 65251, 824, 26940, 32804, 38046, 65441] avg=32768.0000 max=65441 max_rank=7 min=134 min_rank=0 imbalance_ratio=1.997101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[134, 32704, 65251, 824, 26940, 32804, 38046, 65441] avg=32768.0000 max=65441 max_rank=7 min=134 min_rank=0 imbalance_ratio=1.997101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[97799, 7291, 32560, 32654, 32976, 44, 32990, 25830] avg=32768.0000 max=97799 max_rank=0 min=44 min_rank=5 imbalance_ratio=2.984589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[32479, 32704, 33096, 32979, 31890, 32804, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31890 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32345 max_load_reduction_pct=0.494262 moved_to_replicas=134657 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[33096, 33095, 32560, 32654, 32976, 32755, 32990, 32018] avg=32768.0000 max=33096 max_rank=0 min=32018 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64703 max_load_reduction_pct=0.661592 moved_to_replicas=128390 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[134, 32704, 65251, 824, 26940, 32804, 38046, 65441] avg=32768.0000 max=65441 max_rank=7 min=134 min_rank=0 imbalance_ratio=1.997101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[33096, 33095, 32560, 32654, 32976, 32755, 32990, 32018] avg=32768.0000 max=33096 max_rank=0 min=32018 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64703 max_load_reduction_pct=0.661592 moved_to_replicas=128390 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[32479, 32704, 33096, 32979, 31890, 32804, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31890 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32345 max_load_reduction_pct=0.494262 moved_to_replicas=134657 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[32479, 32704, 33096, 32979, 31890, 32804, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31890 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32345 max_load_reduction_pct=0.494262 moved_to_replicas=134657 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[33096, 33095, 32560, 32654, 32976, 32755, 32990, 32018] avg=32768.0000 max=33096 max_rank=0 min=32018 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64703 max_load_reduction_pct=0.661592 moved_to_replicas=128390 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[126, 128, 97262, 14296, 34625, 32732, 49339, 33636] avg=32768.0000 max=97262 max_rank=2 min=126 min_rank=0 imbalance_ratio=2.968201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[32479, 32704, 33096, 32979, 31890, 32804, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31890 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32345 max_load_reduction_pct=0.494262 moved_to_replicas=134657 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[126, 128, 97262, 14296, 34625, 32732, 49339, 33636] avg=32768.0000 max=97262 max_rank=2 min=126 min_rank=0 imbalance_ratio=2.968201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[32833, 32127, 33096, 32069, 33096, 32732, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=32069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64166 max_load_reduction_pct=0.659723 moved_to_replicas=160855 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[134, 32704, 65251, 824, 26940, 32804, 38046, 65441] avg=32768.0000 max=65441 max_rank=7 min=134 min_rank=0 imbalance_ratio=1.997101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[126, 128, 97262, 14296, 34625, 32732, 49339, 33636] avg=32768.0000 max=97262 max_rank=2 min=126 min_rank=0 imbalance_ratio=2.968201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[134, 32704, 65251, 824, 26940, 32804, 38046, 65441] avg=32768.0000 max=65441 max_rank=7 min=134 min_rank=0 imbalance_ratio=1.997101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[126, 128, 97262, 14296, 34625, 32732, 49339, 33636] avg=32768.0000 max=97262 max_rank=2 min=126 min_rank=0 imbalance_ratio=2.968201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[134, 32704, 65251, 824, 26940, 32804, 38046, 65441] avg=32768.0000 max=65441 max_rank=7 min=134 min_rank=0 imbalance_ratio=1.997101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[32833, 32127, 33096, 32069, 33096, 32732, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=32069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64166 max_load_reduction_pct=0.659723 moved_to_replicas=160855 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[126, 128, 97262, 14296, 34625, 32732, 49339, 33636] avg=32768.0000 max=97262 max_rank=2 min=126 min_rank=0 imbalance_ratio=2.968201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[32479, 32704, 33096, 32979, 31890, 32804, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31890 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32345 max_load_reduction_pct=0.494262 moved_to_replicas=134657 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[32833, 32127, 33096, 32069, 33096, 32732, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=32069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64166 max_load_reduction_pct=0.659723 moved_to_replicas=160855 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[32833, 32127, 33096, 32069, 33096, 32732, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=32069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64166 max_load_reduction_pct=0.659723 moved_to_replicas=160855 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[65337, 32918, 32636, 427, 65923, 32503, 32291, 109] avg=32768.0000 max=65923 max_rank=4 min=109 min_rank=7 imbalance_ratio=2.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[32479, 32704, 33096, 32979, 31890, 32804, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31890 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32345 max_load_reduction_pct=0.494262 moved_to_replicas=134657 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[32479, 32704, 33096, 32979, 31890, 32804, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31890 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32345 max_load_reduction_pct=0.494262 moved_to_replicas=134657 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[32833, 32127, 33096, 32069, 33096, 32732, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=32069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64166 max_load_reduction_pct=0.659723 moved_to_replicas=160855 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[65337, 32918, 32636, 427, 65923, 32503, 32291, 109] avg=32768.0000 max=65923 max_rank=4 min=109 min_rank=7 imbalance_ratio=2.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[33096, 32918, 32636, 32762, 33096, 32503, 32291, 32842] avg=32768.0000 max=33096 max_rank=0 min=32291 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32827 max_load_reduction_pct=0.497960 moved_to_replicas=133171 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[126, 128, 97262, 14296, 34625, 32732, 49339, 33636] avg=32768.0000 max=97262 max_rank=2 min=126 min_rank=0 imbalance_ratio=2.968201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[65337, 32918, 32636, 427, 65923, 32503, 32291, 109] avg=32768.0000 max=65923 max_rank=4 min=109 min_rank=7 imbalance_ratio=2.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[65337, 32918, 32636, 427, 65923, 32503, 32291, 109] avg=32768.0000 max=65923 max_rank=4 min=109 min_rank=7 imbalance_ratio=2.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[126, 128, 97262, 14296, 34625, 32732, 49339, 33636] avg=32768.0000 max=97262 max_rank=2 min=126 min_rank=0 imbalance_ratio=2.968201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[33096, 32918, 32636, 32762, 33096, 32503, 32291, 32842] avg=32768.0000 max=33096 max_rank=0 min=32291 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32827 max_load_reduction_pct=0.497960 moved_to_replicas=133171 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[65337, 32918, 32636, 427, 65923, 32503, 32291, 109] avg=32768.0000 max=65923 max_rank=4 min=109 min_rank=7 imbalance_ratio=2.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[32833, 32127, 33096, 32069, 33096, 32732, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=32069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64166 max_load_reduction_pct=0.659723 moved_to_replicas=160855 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[33096, 32918, 32636, 32762, 33096, 32503, 32291, 32842] avg=32768.0000 max=33096 max_rank=0 min=32291 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32827 max_load_reduction_pct=0.497960 moved_to_replicas=133171 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[126, 128, 97262, 14296, 34625, 32732, 49339, 33636] avg=32768.0000 max=97262 max_rank=2 min=126 min_rank=0 imbalance_ratio=2.968201 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[17893, 65232, 33115, 47922, 31902, 634, 8, 65438] avg=32768.0000 max=65438 max_rank=7 min=8 min_rank=6 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[33096, 32918, 32636, 32762, 33096, 32503, 32291, 32842] avg=32768.0000 max=33096 max_rank=0 min=32291 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32827 max_load_reduction_pct=0.497960 moved_to_replicas=133171 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[32833, 32127, 33096, 32069, 33096, 32732, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=32069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64166 max_load_reduction_pct=0.659723 moved_to_replicas=160855 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[33096, 32918, 32636, 32762, 33096, 32503, 32291, 32842] avg=32768.0000 max=33096 max_rank=0 min=32291 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32827 max_load_reduction_pct=0.497960 moved_to_replicas=133171 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[17893, 65232, 33115, 47922, 31902, 634, 8, 65438] avg=32768.0000 max=65438 max_rank=7 min=8 min_rank=6 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[32720, 33096, 33096, 33095, 31921, 32770, 32350, 33096] avg=32768.0000 max=33096 max_rank=1 min=31921 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=158429 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[17893, 65232, 33115, 47922, 31902, 634, 8, 65438] avg=32768.0000 max=65438 max_rank=7 min=8 min_rank=6 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[32833, 32127, 33096, 32069, 33096, 32732, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=32069 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64166 max_load_reduction_pct=0.659723 moved_to_replicas=160855 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[65337, 32918, 32636, 427, 65923, 32503, 32291, 109] avg=32768.0000 max=65923 max_rank=4 min=109 min_rank=7 imbalance_ratio=2.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[17893, 65232, 33115, 47922, 31902, 634, 8, 65438] avg=32768.0000 max=65438 max_rank=7 min=8 min_rank=6 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[32720, 33096, 33096, 33095, 31921, 32770, 32350, 33096] avg=32768.0000 max=33096 max_rank=1 min=31921 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=158429 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[65337, 32918, 32636, 427, 65923, 32503, 32291, 109] avg=32768.0000 max=65923 max_rank=4 min=109 min_rank=7 imbalance_ratio=2.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[17893, 65232, 33115, 47922, 31902, 634, 8, 65438] avg=32768.0000 max=65438 max_rank=7 min=8 min_rank=6 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[32720, 33096, 33096, 33095, 31921, 32770, 32350, 33096] avg=32768.0000 max=33096 max_rank=1 min=31921 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=158429 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[32796, 65405, 65356, 31806, 33602, 281, 174, 32724] avg=32768.0000 max=65405 max_rank=1 min=174 min_rank=6 imbalance_ratio=1.996002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[33096, 32918, 32636, 32762, 33096, 32503, 32291, 32842] avg=32768.0000 max=33096 max_rank=0 min=32291 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32827 max_load_reduction_pct=0.497960 moved_to_replicas=133171 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[32720, 33096, 33096, 33095, 31921, 32770, 32350, 33096] avg=32768.0000 max=33096 max_rank=1 min=31921 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=158429 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[33096, 32918, 32636, 32762, 33096, 32503, 32291, 32842] avg=32768.0000 max=33096 max_rank=0 min=32291 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32827 max_load_reduction_pct=0.497960 moved_to_replicas=133171 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[32796, 65405, 65356, 31806, 33602, 281, 174, 32724] avg=32768.0000 max=65405 max_rank=1 min=174 min_rank=6 imbalance_ratio=1.996002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[32720, 33096, 33096, 33095, 31921, 32770, 32350, 33096] avg=32768.0000 max=33096 max_rank=1 min=31921 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=158429 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[32796, 33096, 33096, 32312, 33096, 32541, 32483, 32724] avg=32768.0000 max=33096 max_rank=1 min=32312 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32309 max_load_reduction_pct=0.493984 moved_to_replicas=128969 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[65337, 32918, 32636, 427, 65923, 32503, 32291, 109] avg=32768.0000 max=65923 max_rank=4 min=109 min_rank=7 imbalance_ratio=2.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[32796, 65405, 65356, 31806, 33602, 281, 174, 32724] avg=32768.0000 max=65405 max_rank=1 min=174 min_rank=6 imbalance_ratio=1.996002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[32796, 33096, 33096, 32312, 33096, 32541, 32483, 32724] avg=32768.0000 max=33096 max_rank=1 min=32312 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32309 max_load_reduction_pct=0.493984 moved_to_replicas=128969 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[32796, 65405, 65356, 31806, 33602, 281, 174, 32724] avg=32768.0000 max=65405 max_rank=1 min=174 min_rank=6 imbalance_ratio=1.996002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[17893, 65232, 33115, 47922, 31902, 634, 8, 65438] avg=32768.0000 max=65438 max_rank=7 min=8 min_rank=6 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[17893, 65232, 33115, 47922, 31902, 634, 8, 65438] avg=32768.0000 max=65438 max_rank=7 min=8 min_rank=6 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[33096, 32918, 32636, 32762, 33096, 32503, 32291, 32842] avg=32768.0000 max=33096 max_rank=0 min=32291 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32827 max_load_reduction_pct=0.497960 moved_to_replicas=133171 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[32796, 33096, 33096, 32312, 33096, 32541, 32483, 32724] avg=32768.0000 max=33096 max_rank=1 min=32312 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32309 max_load_reduction_pct=0.493984 moved_to_replicas=128969 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[32796, 65405, 65356, 31806, 33602, 281, 174, 32724] avg=32768.0000 max=65405 max_rank=1 min=174 min_rank=6 imbalance_ratio=1.996002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[30453, 32705, 65212, 65906, 56, 33211, 32910, 1691] avg=32768.0000 max=65906 max_rank=3 min=56 min_rank=4 imbalance_ratio=2.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[32796, 33096, 33096, 32312, 33096, 32541, 32483, 32724] avg=32768.0000 max=33096 max_rank=1 min=32312 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32309 max_load_reduction_pct=0.493984 moved_to_replicas=128969 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[32720, 33096, 33096, 33095, 31921, 32770, 32350, 33096] avg=32768.0000 max=33096 max_rank=1 min=31921 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=158429 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[32720, 33096, 33096, 33095, 31921, 32770, 32350, 33096] avg=32768.0000 max=33096 max_rank=1 min=31921 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=158429 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[30453, 32705, 65212, 65906, 56, 33211, 32910, 1691] avg=32768.0000 max=65906 max_rank=3 min=56 min_rank=4 imbalance_ratio=2.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[32796, 33096, 33096, 32312, 33096, 32541, 32483, 32724] avg=32768.0000 max=33096 max_rank=1 min=32312 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32309 max_load_reduction_pct=0.493984 moved_to_replicas=128969 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[31382, 32705, 33096, 33096, 32763, 33096, 32910, 33096] avg=32768.0000 max=33096 max_rank=2 min=31382 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32810 max_load_reduction_pct=0.497830 moved_to_replicas=133702 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[30453, 32705, 65212, 65906, 56, 33211, 32910, 1691] avg=32768.0000 max=65906 max_rank=3 min=56 min_rank=4 imbalance_ratio=2.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[17893, 65232, 33115, 47922, 31902, 634, 8, 65438] avg=32768.0000 max=65438 max_rank=7 min=8 min_rank=6 imbalance_ratio=1.997009 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[31382, 32705, 33096, 33096, 32763, 33096, 32910, 33096] avg=32768.0000 max=33096 max_rank=2 min=31382 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32810 max_load_reduction_pct=0.497830 moved_to_replicas=133702 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[30453, 32705, 65212, 65906, 56, 33211, 32910, 1691] avg=32768.0000 max=65906 max_rank=3 min=56 min_rank=4 imbalance_ratio=2.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[32796, 65405, 65356, 31806, 33602, 281, 174, 32724] avg=32768.0000 max=65405 max_rank=1 min=174 min_rank=6 imbalance_ratio=1.996002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[32796, 65405, 65356, 31806, 33602, 281, 174, 32724] avg=32768.0000 max=65405 max_rank=1 min=174 min_rank=6 imbalance_ratio=1.996002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[31382, 32705, 33096, 33096, 32763, 33096, 32910, 33096] avg=32768.0000 max=33096 max_rank=2 min=31382 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32810 max_load_reduction_pct=0.497830 moved_to_replicas=133702 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[32720, 33096, 33096, 33095, 31921, 32770, 32350, 33096] avg=32768.0000 max=33096 max_rank=1 min=31921 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32342 max_load_reduction_pct=0.494239 moved_to_replicas=158429 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[30453, 32705, 65212, 65906, 56, 33211, 32910, 1691] avg=32768.0000 max=65906 max_rank=3 min=56 min_rank=4 imbalance_ratio=2.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[31382, 32705, 33096, 33096, 32763, 33096, 32910, 33096] avg=32768.0000 max=33096 max_rank=2 min=31382 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32810 max_load_reduction_pct=0.497830 moved_to_replicas=133702 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[32796, 33096, 33096, 32312, 33096, 32541, 32483, 32724] avg=32768.0000 max=33096 max_rank=1 min=32312 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32309 max_load_reduction_pct=0.493984 moved_to_replicas=128969 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[32796, 33096, 33096, 32312, 33096, 32541, 32483, 32724] avg=32768.0000 max=33096 max_rank=1 min=32312 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32309 max_load_reduction_pct=0.493984 moved_to_replicas=128969 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[31382, 32705, 33096, 33096, 32763, 33096, 32910, 33096] avg=32768.0000 max=33096 max_rank=2 min=31382 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32810 max_load_reduction_pct=0.497830 moved_to_replicas=133702 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[30453, 32705, 65212, 65906, 56, 33211, 32910, 1691] avg=32768.0000 max=65906 max_rank=3 min=56 min_rank=4 imbalance_ratio=2.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[30453, 32705, 65212, 65906, 56, 33211, 32910, 1691] avg=32768.0000 max=65906 max_rank=3 min=56 min_rank=4 imbalance_ratio=2.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[32796, 65405, 65356, 31806, 33602, 281, 174, 32724] avg=32768.0000 max=65405 max_rank=1 min=174 min_rank=6 imbalance_ratio=1.996002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[31382, 32705, 33096, 33096, 32763, 33096, 32910, 33096] avg=32768.0000 max=33096 max_rank=2 min=31382 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32810 max_load_reduction_pct=0.497830 moved_to_replicas=133702 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[31382, 32705, 33096, 33096, 32763, 33096, 32910, 33096] avg=32768.0000 max=33096 max_rank=2 min=31382 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32810 max_load_reduction_pct=0.497830 moved_to_replicas=133702 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[32796, 33096, 33096, 32312, 33096, 32541, 32483, 32724] avg=32768.0000 max=33096 max_rank=1 min=32312 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32309 max_load_reduction_pct=0.493984 moved_to_replicas=128969 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[30453, 32705, 65212, 65906, 56, 33211, 32910, 1691] avg=32768.0000 max=65906 max_rank=3 min=56 min_rank=4 imbalance_ratio=2.011292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[31382, 32705, 33096, 33096, 32763, 33096, 32910, 33096] avg=32768.0000 max=33096 max_rank=2 min=31382 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32810 max_load_reduction_pct=0.497830 moved_to_replicas=133702 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=32 total_expected_recv=1048039 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=53632 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=32 total_expected_recv=1048677 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=53760 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=32 total_expected_recv=1049540 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=52736 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=32 total_expected_recv=1050492 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=52992 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=32 total_expected_recv=1043619 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=59008 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=32 total_expected_recv=1048966 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=53248 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=32 total_expected_recv=1050115 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=51968 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=32 total_expected_recv=1049160 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=52992 peak_demand=33097 required_capacity=1.010040 + [2026-08-30 05:11:42] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 570.1 | throughput per GPU (TFLOP/s/GPU): 584.8 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.322896E+01 | loss scale: 1.0 | grad norm: 31.661 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=34816 expected_recv=61648 expected_overflow=0 exact_recv_per_rank=[33097, 30472, 33096, 33095, 33095, 33094, 33096, 33099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 30472, 33096, 33095, 33095, 33094, 33096, 33099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 30472, 33096, 33095, 33095, 33094, 33096, 33099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33097, 30472, 33096, 33095, 33095, 33094, 33096, 33099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=34816 expected_recv=66189 expected_overflow=0 exact_recv_per_rank=[33097, 30472, 33096, 33095, 33095, 33094, 33096, 33099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=34816 expected_recv=66106 expected_overflow=0 exact_recv_per_rank=[33097, 30472, 33096, 33095, 33095, 33094, 33096, 33099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=34816 expected_recv=65577 expected_overflow=0 exact_recv_per_rank=[33097, 30472, 33096, 33095, 33095, 33094, 33096, 33099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=34816 expected_recv=61648 actual_valid_recv=62336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33097, 30472, 33096, 33095, 33095, 33094, 33096, 33099] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=34816 expected_recv=66189 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=62336 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=34816 expected_recv=66106 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=34816 expected_recv=65577 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31169, 32790, 32705, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=34816 expected_recv=64265 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31169, 32790, 32705, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31169, 32790, 32705, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31169, 32790, 32705, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=34816 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31169, 32790, 32705, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=34816 expected_recv=64265 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=34816 expected_recv=64408 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31169, 32790, 32705, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31169, 32790, 32705, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=34816 expected_recv=65530 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=34816 expected_recv=65317 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31169, 32790, 32705, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=34816 expected_recv=64408 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=34816 expected_recv=65317 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=34816 expected_recv=65648 expected_overflow=0 exact_recv_per_rank=[31826, 32851, 32710, 33097, 32373, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=34816 expected_recv=65428 expected_overflow=0 exact_recv_per_rank=[31826, 32851, 32710, 33097, 32373, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=34816 expected_recv=65648 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=34816 expected_recv=65428 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=34816 expected_recv=63463 expected_overflow=0 exact_recv_per_rank=[31826, 32851, 32710, 33097, 32373, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31826, 32851, 32710, 33097, 32373, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31826, 32851, 32710, 33097, 32373, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=34816 expected_recv=63463 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=34816 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[31826, 32851, 32710, 33097, 32373, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31826, 32851, 32710, 33097, 32373, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=34816 expected_recv=64981 expected_overflow=0 exact_recv_per_rank=[31826, 32851, 32710, 33097, 32373, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=64640 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=34816 expected_recv=66190 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=67200 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=34816 expected_recv=64981 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=34816 expected_recv=65765 expected_overflow=0 exact_recv_per_rank=[32742, 32927, 33096, 33096, 33096, 31902, 32493, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32742, 32927, 33096, 33096, 33096, 31902, 32493, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=34816 expected_recv=65765 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=34816 expected_recv=65512 expected_overflow=0 exact_recv_per_rank=[32742, 32927, 33096, 33096, 33096, 31902, 32493, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=34816 expected_recv=63797 expected_overflow=0 exact_recv_per_rank=[32742, 32927, 33096, 33096, 33096, 31902, 32493, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32742, 32927, 33096, 33096, 33096, 31902, 32493, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=34816 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[32742, 32927, 33096, 33096, 33096, 31902, 32493, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=34816 expected_recv=65512 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=34816 expected_recv=63797 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32742, 32927, 33096, 33096, 33096, 31902, 32493, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=34816 expected_recv=65099 expected_overflow=0 exact_recv_per_rank=[32742, 32927, 33096, 33096, 33096, 31902, 32493, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=34816 expected_recv=65539 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=64768 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=34816 expected_recv=65099 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=34816 expected_recv=65344 expected_overflow=0 exact_recv_per_rank=[33096, 32737, 32074, 32676, 33094, 33098, 32272, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=34816 expected_recv=64014 expected_overflow=0 exact_recv_per_rank=[33096, 32737, 32074, 32676, 33094, 33098, 32272, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=34816 expected_recv=65344 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=34816 expected_recv=64014 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32737, 32074, 32676, 33094, 33098, 32272, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=34816 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33096, 32737, 32074, 32676, 33094, 33098, 32272, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=64768 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=34816 expected_recv=65371 expected_overflow=0 exact_recv_per_rank=[33096, 32737, 32074, 32676, 33094, 33098, 32272, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=34816 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33096, 32737, 32074, 32676, 33094, 33098, 32272, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 32737, 32074, 32676, 33094, 33098, 32272, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=34816 expected_recv=66194 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=34816 expected_recv=65371 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=34816 expected_recv=66194 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=34816 expected_recv=64788 expected_overflow=0 exact_recv_per_rank=[33096, 32737, 32074, 32676, 33094, 33098, 32272, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=34816 expected_recv=64788 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=34816 expected_recv=64351 expected_overflow=0 exact_recv_per_rank=[33096, 32542, 32768, 32701, 33096, 32066, 32779, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=34816 expected_recv=65572 expected_overflow=0 exact_recv_per_rank=[33096, 32542, 32768, 32701, 33096, 32066, 32779, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=34816 expected_recv=64351 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=34816 expected_recv=65572 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32542, 32768, 32701, 33096, 32066, 32779, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=34816 expected_recv=64802 expected_overflow=0 exact_recv_per_rank=[33096, 32542, 32768, 32701, 33096, 32066, 32779, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=34816 expected_recv=65441 expected_overflow=0 exact_recv_per_rank=[33096, 32542, 32768, 32701, 33096, 32066, 32779, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32542, 32768, 32701, 33096, 32066, 32779, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32542, 32768, 32701, 33096, 32066, 32779, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=34816 expected_recv=64802 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=34816 expected_recv=65441 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=34816 expected_recv=65615 expected_overflow=0 exact_recv_per_rank=[32762, 32858, 33096, 32770, 32775, 32439, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=34816 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[33096, 32542, 32768, 32701, 33096, 32066, 32779, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32762, 32858, 33096, 32770, 32775, 32439, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=34816 expected_recv=65615 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=34816 expected_recv=65546 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=34816 expected_recv=65508 expected_overflow=0 exact_recv_per_rank=[32762, 32858, 33096, 32770, 32775, 32439, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=34816 expected_recv=64944 expected_overflow=0 exact_recv_per_rank=[32762, 32858, 33096, 32770, 32775, 32439, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=34816 expected_recv=65510 expected_overflow=0 exact_recv_per_rank=[32762, 32858, 33096, 32770, 32775, 32439, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=34816 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[32762, 32858, 33096, 32770, 32775, 32439, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=34816 expected_recv=65508 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=34816 expected_recv=64781 expected_overflow=0 exact_recv_per_rank=[32762, 32858, 33096, 32770, 32775, 32439, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=34816 expected_recv=64944 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=34816 expected_recv=65510 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=34816 expected_recv=65546 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=34816 expected_recv=64781 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=34816 expected_recv=63094 expected_overflow=0 exact_recv_per_rank=[32765, 31405, 32776, 32814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=34816 expected_recv=65568 expected_overflow=0 exact_recv_per_rank=[32765, 31405, 32776, 32814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32762, 32858, 33096, 32770, 32775, 32439, 33096, 32348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=34816 expected_recv=63094 actual_valid_recv=64128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=34816 expected_recv=65568 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=64128 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32765, 31405, 32776, 32814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=34816 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[32765, 31405, 32776, 32814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=34816 expected_recv=65606 expected_overflow=0 exact_recv_per_rank=[32765, 31405, 32776, 32814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=34816 expected_recv=65899 expected_overflow=0 exact_recv_per_rank=[32765, 31405, 32776, 32814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32765, 31405, 32776, 32814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=34816 expected_recv=65545 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=34816 expected_recv=65606 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=34816 expected_recv=65899 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=34816 expected_recv=64629 expected_overflow=0 exact_recv_per_rank=[33096, 32283, 32767, 32576, 32780, 33096, 32777, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=34816 expected_recv=65503 expected_overflow=0 exact_recv_per_rank=[33096, 32283, 32767, 32576, 32780, 33096, 32777, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32765, 31405, 32776, 32814, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=34816 expected_recv=64629 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=34816 expected_recv=65503 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32283, 32767, 32576, 32780, 33096, 32777, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=34816 expected_recv=65979 expected_overflow=0 exact_recv_per_rank=[33096, 32283, 32767, 32576, 32780, 33096, 32777, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=34816 expected_recv=65583 expected_overflow=0 exact_recv_per_rank=[33096, 32283, 32767, 32576, 32780, 33096, 32777, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=34816 expected_recv=65362 expected_overflow=0 exact_recv_per_rank=[33096, 32283, 32767, 32576, 32780, 33096, 32777, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=34816 expected_recv=65515 expected_overflow=0 exact_recv_per_rank=[33096, 32283, 32767, 32576, 32780, 33096, 32777, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=34816 expected_recv=65979 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=34816 expected_recv=65583 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=34816 expected_recv=65362 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=34816 expected_recv=65515 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32302, 32769, 32838, 32182, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=34816 expected_recv=64728 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32302, 32769, 32838, 32182, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=34816 expected_recv=64728 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=34816 expected_recv=65525 expected_overflow=0 exact_recv_per_rank=[33096, 32283, 32767, 32576, 32780, 33096, 32777, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=34816 expected_recv=64595 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32302, 32769, 32838, 32182, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=34816 expected_recv=65912 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32302, 32769, 32838, 32182, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=34816 expected_recv=65525 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=34816 expected_recv=64595 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=34816 expected_recv=65643 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32302, 32769, 32838, 32182, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=34816 expected_recv=65912 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=34816 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32302, 32769, 32838, 32182, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65024 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32302, 32769, 32838, 32182, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=34816 expected_recv=65643 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=34816 expected_recv=65530 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 33096, 33096, 32400, 32443, 32764, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=34816 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 33096, 33096, 32400, 32443, 32764, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=34816 expected_recv=65523 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=34816 expected_recv=65496 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32302, 32769, 32838, 32182, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=34816 expected_recv=64867 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 33096, 33096, 32400, 32443, 32764, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=34816 expected_recv=65496 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=34816 expected_recv=64867 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=34816 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 33096, 33096, 32400, 32443, 32764, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=34816 expected_recv=64769 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 33096, 33096, 32400, 32443, 32764, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 33096, 33096, 32400, 32443, 32764, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=34816 expected_recv=65544 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=34816 expected_recv=65646 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 33096, 33096, 32400, 32443, 32764, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=34816 expected_recv=64769 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=34816 expected_recv=65646 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=34816 expected_recv=65561 expected_overflow=0 exact_recv_per_rank=[32787, 32394, 32829, 33096, 33096, 32434, 32412, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=34816 expected_recv=64905 expected_overflow=0 exact_recv_per_rank=[32787, 32394, 32829, 33096, 33096, 32434, 32412, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=34816 expected_recv=65561 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=34816 expected_recv=64905 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=34816 expected_recv=65555 expected_overflow=0 exact_recv_per_rank=[32766, 32768, 33096, 33096, 32400, 32443, 32764, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=34816 expected_recv=65193 expected_overflow=0 exact_recv_per_rank=[32787, 32394, 32829, 33096, 33096, 32434, 32412, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=34816 expected_recv=65608 expected_overflow=0 exact_recv_per_rank=[32787, 32394, 32829, 33096, 33096, 32434, 32412, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=34816 expected_recv=65555 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32787, 32394, 32829, 33096, 33096, 32434, 32412, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=34816 expected_recv=65193 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=34816 expected_recv=65608 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32787, 32394, 32829, 33096, 33096, 32434, 32412, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32787, 32394, 32829, 33096, 33096, 32434, 32412, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=34816 expected_recv=64837 expected_overflow=0 exact_recv_per_rank=[32194, 32772, 32522, 33096, 32930, 32438, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=34816 expected_recv=65596 expected_overflow=0 exact_recv_per_rank=[32194, 32772, 32522, 33096, 32930, 32438, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=34816 expected_recv=64837 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=34816 expected_recv=65596 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=34816 expected_recv=64861 expected_overflow=0 exact_recv_per_rank=[32194, 32772, 32522, 33096, 32930, 32438, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=34816 expected_recv=64445 expected_overflow=0 exact_recv_per_rank=[32787, 32394, 32829, 33096, 33096, 32434, 32412, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=34816 expected_recv=64794 expected_overflow=0 exact_recv_per_rank=[32194, 32772, 32522, 33096, 32930, 32438, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=34816 expected_recv=65624 expected_overflow=0 exact_recv_per_rank=[32194, 32772, 32522, 33096, 32930, 32438, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=34816 expected_recv=64861 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=34816 expected_recv=64445 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32194, 32772, 32522, 33096, 32930, 32438, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=34816 expected_recv=64794 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32194, 32772, 32522, 33096, 32930, 32438, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=34816 expected_recv=65624 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=34816 expected_recv=64751 expected_overflow=0 exact_recv_per_rank=[32542, 32769, 32338, 33096, 32438, 33096, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=34816 expected_recv=65595 expected_overflow=0 exact_recv_per_rank=[32542, 32769, 32338, 33096, 32438, 33096, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=34816 expected_recv=64751 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=34816 expected_recv=65595 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32542, 32769, 32338, 33096, 32438, 33096, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32194, 32772, 32522, 33096, 32930, 32438, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=34816 expected_recv=64969 expected_overflow=0 exact_recv_per_rank=[32542, 32769, 32338, 33096, 32438, 33096, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=34816 expected_recv=64905 expected_overflow=0 exact_recv_per_rank=[32542, 32769, 32338, 33096, 32438, 33096, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=34816 expected_recv=64969 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32542, 32769, 32338, 33096, 32438, 33096, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=34816 expected_recv=64905 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32542, 32769, 32338, 33096, 32438, 33096, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=34816 expected_recv=65009 expected_overflow=0 exact_recv_per_rank=[32442, 32765, 32445, 32762, 33096, 32767, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=34816 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[32442, 32765, 32445, 32762, 33096, 32767, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=34816 expected_recv=65009 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=34816 expected_recv=65533 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=34816 expected_recv=65412 expected_overflow=0 exact_recv_per_rank=[32442, 32765, 32445, 32762, 33096, 32767, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=34816 expected_recv=65492 expected_overflow=0 exact_recv_per_rank=[32542, 32769, 32338, 33096, 32438, 33096, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=34816 expected_recv=65412 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=34816 expected_recv=64871 expected_overflow=0 exact_recv_per_rank=[32442, 32765, 32445, 32762, 33096, 32767, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32442, 32765, 32445, 32762, 33096, 32767, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=34816 expected_recv=65492 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=34816 expected_recv=65513 expected_overflow=0 exact_recv_per_rank=[32442, 32765, 32445, 32762, 33096, 32767, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=34816 expected_recv=64871 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32442, 32765, 32445, 32762, 33096, 32767, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=34816 expected_recv=65463 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32764, 32443, 32766, 32764, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32764, 32443, 32766, 32764, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=34816 expected_recv=65513 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=34816 expected_recv=65463 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=34816 expected_recv=64937 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32764, 32443, 32766, 32764, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=34816 expected_recv=65566 expected_overflow=0 exact_recv_per_rank=[32442, 32765, 32445, 32762, 33096, 32767, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=34816 expected_recv=64937 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=34816 expected_recv=65595 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32764, 32443, 32766, 32764, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=34816 expected_recv=65508 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32764, 32443, 32766, 32764, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=34816 expected_recv=65566 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=34816 expected_recv=65022 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32764, 32443, 32766, 32764, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=34816 expected_recv=65595 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=34816 expected_recv=65508 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32764, 32443, 32766, 32764, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=34816 expected_recv=65022 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[48768, 29468, 54829, 15614, 38746, 40993, 13310, 20416] avg=32768.0000 max=54829 max_rank=2 min=13310 min_rank=6 imbalance_ratio=1.673248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[48768, 29468, 54829, 15614, 38746, 40993, 13310, 20416] avg=32768.0000 max=54829 max_rank=2 min=13310 min_rank=6 imbalance_ratio=1.673248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[48768, 29468, 54829, 15614, 38746, 40993, 13310, 20416] avg=32768.0000 max=54829 max_rank=2 min=13310 min_rank=6 imbalance_ratio=1.673248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[33097, 30472, 33096, 33095, 33095, 33094, 33096, 33099] avg=32768.0000 max=33099 max_rank=7 min=30472 min_rank=1 imbalance_ratio=1.010101 max_load_reduction=21730 max_load_reduction_pct=0.396323 moved_to_replicas=98834 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=34816 expected_recv=65379 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32764, 32443, 32766, 32764, 32450, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[33097, 30472, 33096, 33095, 33095, 33094, 33096, 33099] avg=32768.0000 max=33099 max_rank=7 min=30472 min_rank=1 imbalance_ratio=1.010101 max_load_reduction=21730 max_load_reduction_pct=0.396323 moved_to_replicas=98834 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[48768, 29468, 54829, 15614, 38746, 40993, 13310, 20416] avg=32768.0000 max=54829 max_rank=2 min=13310 min_rank=6 imbalance_ratio=1.673248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=34816 expected_recv=65379 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[48768, 29468, 54829, 15614, 38746, 40993, 13310, 20416] avg=32768.0000 max=54829 max_rank=2 min=13310 min_rank=6 imbalance_ratio=1.673248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[48768, 29468, 54829, 15614, 38746, 40993, 13310, 20416] avg=32768.0000 max=54829 max_rank=2 min=13310 min_rank=6 imbalance_ratio=1.673248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[33097, 30472, 33096, 33095, 33095, 33094, 33096, 33099] avg=32768.0000 max=33099 max_rank=7 min=30472 min_rank=1 imbalance_ratio=1.010101 max_load_reduction=21730 max_load_reduction_pct=0.396323 moved_to_replicas=98834 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[48768, 29468, 54829, 15614, 38746, 40993, 13310, 20416] avg=32768.0000 max=54829 max_rank=2 min=13310 min_rank=6 imbalance_ratio=1.673248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[36142, 65398, 13575, 32790, 32705, 45589, 11814, 24131] avg=32768.0000 max=65398 max_rank=1 min=11814 min_rank=6 imbalance_ratio=1.995789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[33097, 30472, 33096, 33095, 33095, 33094, 33096, 33099] avg=32768.0000 max=33099 max_rank=7 min=30472 min_rank=1 imbalance_ratio=1.010101 max_load_reduction=21730 max_load_reduction_pct=0.396323 moved_to_replicas=98834 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[33097, 30472, 33096, 33095, 33095, 33094, 33096, 33099] avg=32768.0000 max=33099 max_rank=7 min=30472 min_rank=1 imbalance_ratio=1.010101 max_load_reduction=21730 max_load_reduction_pct=0.396323 moved_to_replicas=98834 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[36142, 65398, 13575, 32790, 32705, 45589, 11814, 24131] avg=32768.0000 max=65398 max_rank=1 min=11814 min_rank=6 imbalance_ratio=1.995789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[33097, 30472, 33096, 33095, 33095, 33094, 33096, 33099] avg=32768.0000 max=33099 max_rank=7 min=30472 min_rank=1 imbalance_ratio=1.010101 max_load_reduction=21730 max_load_reduction_pct=0.396323 moved_to_replicas=98834 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[33097, 30472, 33096, 33095, 33095, 33094, 33096, 33099] avg=32768.0000 max=33099 max_rank=7 min=30472 min_rank=1 imbalance_ratio=1.010101 max_load_reduction=21730 max_load_reduction_pct=0.396323 moved_to_replicas=98834 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[36142, 65398, 13575, 32790, 32705, 45589, 11814, 24131] avg=32768.0000 max=65398 max_rank=1 min=11814 min_rank=6 imbalance_ratio=1.995789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 31169, 32790, 32705, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31169 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32302 max_load_reduction_pct=0.493929 moved_to_replicas=96491 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[48768, 29468, 54829, 15614, 38746, 40993, 13310, 20416] avg=32768.0000 max=54829 max_rank=2 min=13310 min_rank=6 imbalance_ratio=1.673248 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 31169, 32790, 32705, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31169 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32302 max_load_reduction_pct=0.493929 moved_to_replicas=96491 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[36142, 65398, 13575, 32790, 32705, 45589, 11814, 24131] avg=32768.0000 max=65398 max_rank=1 min=11814 min_rank=6 imbalance_ratio=1.995789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[36142, 65398, 13575, 32790, 32705, 45589, 11814, 24131] avg=32768.0000 max=65398 max_rank=1 min=11814 min_rank=6 imbalance_ratio=1.995789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[36142, 65398, 13575, 32790, 32705, 45589, 11814, 24131] avg=32768.0000 max=65398 max_rank=1 min=11814 min_rank=6 imbalance_ratio=1.995789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 31169, 32790, 32705, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31169 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32302 max_load_reduction_pct=0.493929 moved_to_replicas=96491 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[33097, 30472, 33096, 33095, 33095, 33094, 33096, 33099] avg=32768.0000 max=33099 max_rank=7 min=30472 min_rank=1 imbalance_ratio=1.010101 max_load_reduction=21730 max_load_reduction_pct=0.396323 moved_to_replicas=98834 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[36142, 65398, 13575, 32790, 32705, 45589, 11814, 24131] avg=32768.0000 max=65398 max_rank=1 min=11814 min_rank=6 imbalance_ratio=1.995789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 31169, 32790, 32705, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31169 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32302 max_load_reduction_pct=0.493929 moved_to_replicas=96491 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[53, 32851, 32710, 57271, 32373, 372, 60894, 45620] avg=32768.0000 max=60894 max_rank=6 min=53 min_rank=0 imbalance_ratio=1.858337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 31169, 32790, 32705, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31169 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32302 max_load_reduction_pct=0.493929 moved_to_replicas=96491 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[53, 32851, 32710, 57271, 32373, 372, 60894, 45620] avg=32768.0000 max=60894 max_rank=6 min=53 min_rank=0 imbalance_ratio=1.858337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 31169, 32790, 32705, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31169 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32302 max_load_reduction_pct=0.493929 moved_to_replicas=96491 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 31169, 32790, 32705, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31169 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32302 max_load_reduction_pct=0.493929 moved_to_replicas=96491 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[53, 32851, 32710, 57271, 32373, 372, 60894, 45620] avg=32768.0000 max=60894 max_rank=6 min=53 min_rank=0 imbalance_ratio=1.858337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[31826, 32851, 32710, 33097, 32373, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31826 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27797 max_load_reduction_pct=0.456482 moved_to_replicas=129009 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[31826, 32851, 32710, 33097, 32373, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31826 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27797 max_load_reduction_pct=0.456482 moved_to_replicas=129009 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[36142, 65398, 13575, 32790, 32705, 45589, 11814, 24131] avg=32768.0000 max=65398 max_rank=1 min=11814 min_rank=6 imbalance_ratio=1.995789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[53, 32851, 32710, 57271, 32373, 372, 60894, 45620] avg=32768.0000 max=60894 max_rank=6 min=53 min_rank=0 imbalance_ratio=1.858337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[53, 32851, 32710, 57271, 32373, 372, 60894, 45620] avg=32768.0000 max=60894 max_rank=6 min=53 min_rank=0 imbalance_ratio=1.858337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[53, 32851, 32710, 57271, 32373, 372, 60894, 45620] avg=32768.0000 max=60894 max_rank=6 min=53 min_rank=0 imbalance_ratio=1.858337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[31826, 32851, 32710, 33097, 32373, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31826 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27797 max_load_reduction_pct=0.456482 moved_to_replicas=129009 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 31169, 32790, 32705, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31169 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32302 max_load_reduction_pct=0.493929 moved_to_replicas=96491 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[53, 32851, 32710, 57271, 32373, 372, 60894, 45620] avg=32768.0000 max=60894 max_rank=6 min=53 min_rank=0 imbalance_ratio=1.858337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[32742, 32927, 4733, 62676, 65492, 30685, 97, 32792] avg=32768.0000 max=65492 max_rank=4 min=97 min_rank=6 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[32742, 32927, 4733, 62676, 65492, 30685, 97, 32792] avg=32768.0000 max=65492 max_rank=4 min=97 min_rank=6 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[31826, 32851, 32710, 33097, 32373, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31826 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27797 max_load_reduction_pct=0.456482 moved_to_replicas=129009 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[31826, 32851, 32710, 33097, 32373, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31826 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27797 max_load_reduction_pct=0.456482 moved_to_replicas=129009 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[31826, 32851, 32710, 33097, 32373, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31826 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27797 max_load_reduction_pct=0.456482 moved_to_replicas=129009 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[32742, 32927, 4733, 62676, 65492, 30685, 97, 32792] avg=32768.0000 max=65492 max_rank=4 min=97 min_rank=6 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[31826, 32851, 32710, 33097, 32373, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31826 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27797 max_load_reduction_pct=0.456482 moved_to_replicas=129009 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[32742, 32927, 33096, 33096, 33096, 31902, 32493, 32792] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=122237 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[32742, 32927, 33096, 33096, 33096, 31902, 32493, 32792] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=122237 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[53, 32851, 32710, 57271, 32373, 372, 60894, 45620] avg=32768.0000 max=60894 max_rank=6 min=53 min_rank=0 imbalance_ratio=1.858337 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[32742, 32927, 4733, 62676, 65492, 30685, 97, 32792] avg=32768.0000 max=65492 max_rank=4 min=97 min_rank=6 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[32742, 32927, 4733, 62676, 65492, 30685, 97, 32792] avg=32768.0000 max=65492 max_rank=4 min=97 min_rank=6 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[32742, 32927, 33096, 33096, 33096, 31902, 32493, 32792] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=122237 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[32742, 32927, 4733, 62676, 65492, 30685, 97, 32792] avg=32768.0000 max=65492 max_rank=4 min=97 min_rank=6 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[31826, 32851, 32710, 33097, 32373, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=3 min=31826 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=27797 max_load_reduction_pct=0.456482 moved_to_replicas=129009 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[32742, 32927, 4733, 62676, 65492, 30685, 97, 32792] avg=32768.0000 max=65492 max_rank=4 min=97 min_rank=6 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[65136, 32737, 24373, 32676, 528, 56848, 232, 49614] avg=32768.0000 max=65136 max_rank=0 min=232 min_rank=6 imbalance_ratio=1.987793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[65136, 32737, 24373, 32676, 528, 56848, 232, 49614] avg=32768.0000 max=65136 max_rank=0 min=232 min_rank=6 imbalance_ratio=1.987793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[32742, 32927, 33096, 33096, 33096, 31902, 32493, 32792] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=122237 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[32742, 32927, 33096, 33096, 33096, 31902, 32493, 32792] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=122237 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[32742, 32927, 33096, 33096, 33096, 31902, 32493, 32792] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=122237 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[65136, 32737, 24373, 32676, 528, 56848, 232, 49614] avg=32768.0000 max=65136 max_rank=0 min=232 min_rank=6 imbalance_ratio=1.987793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[32742, 32927, 33096, 33096, 33096, 31902, 32493, 32792] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=122237 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32737, 32074, 32676, 33094, 33098, 32272, 33097] avg=32768.0000 max=33098 max_rank=5 min=32074 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32038 max_load_reduction_pct=0.491863 moved_to_replicas=142022 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32737, 32074, 32676, 33094, 33098, 32272, 33097] avg=32768.0000 max=33098 max_rank=5 min=32074 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32038 max_load_reduction_pct=0.491863 moved_to_replicas=142022 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[32742, 32927, 4733, 62676, 65492, 30685, 97, 32792] avg=32768.0000 max=65492 max_rank=4 min=97 min_rank=6 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[65136, 32737, 24373, 32676, 528, 56848, 232, 49614] avg=32768.0000 max=65136 max_rank=0 min=232 min_rank=6 imbalance_ratio=1.987793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32737, 32074, 32676, 33094, 33098, 32272, 33097] avg=32768.0000 max=33098 max_rank=5 min=32074 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32038 max_load_reduction_pct=0.491863 moved_to_replicas=142022 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[65136, 32737, 24373, 32676, 528, 56848, 232, 49614] avg=32768.0000 max=65136 max_rank=0 min=232 min_rank=6 imbalance_ratio=1.987793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[65136, 32737, 24373, 32676, 528, 56848, 232, 49614] avg=32768.0000 max=65136 max_rank=0 min=232 min_rank=6 imbalance_ratio=1.987793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[32742, 32927, 33096, 33096, 33096, 31902, 32493, 32792] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=122237 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[65136, 32737, 24373, 32676, 528, 56848, 232, 49614] avg=32768.0000 max=65136 max_rank=0 min=232 min_rank=6 imbalance_ratio=1.987793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[33372, 1, 32768, 263, 97238, 189, 32779, 65534] avg=32768.0000 max=97238 max_rank=4 min=1 min_rank=1 imbalance_ratio=2.967468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[33372, 1, 32768, 263, 97238, 189, 32779, 65534] avg=32768.0000 max=97238 max_rank=4 min=1 min_rank=1 imbalance_ratio=2.967468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32737, 32074, 32676, 33094, 33098, 32272, 33097] avg=32768.0000 max=33098 max_rank=5 min=32074 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32038 max_load_reduction_pct=0.491863 moved_to_replicas=142022 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32737, 32074, 32676, 33094, 33098, 32272, 33097] avg=32768.0000 max=33098 max_rank=5 min=32074 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32038 max_load_reduction_pct=0.491863 moved_to_replicas=142022 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32737, 32074, 32676, 33094, 33098, 32272, 33097] avg=32768.0000 max=33098 max_rank=5 min=32074 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32038 max_load_reduction_pct=0.491863 moved_to_replicas=142022 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[33372, 1, 32768, 263, 97238, 189, 32779, 65534] avg=32768.0000 max=97238 max_rank=4 min=1 min_rank=1 imbalance_ratio=2.967468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32737, 32074, 32676, 33094, 33098, 32272, 33097] avg=32768.0000 max=33098 max_rank=5 min=32074 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32038 max_load_reduction_pct=0.491863 moved_to_replicas=142022 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[33096, 32542, 32768, 32701, 33096, 32066, 32779, 33096] avg=32768.0000 max=33096 max_rank=0 min=32066 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64142 max_load_reduction_pct=0.659639 moved_to_replicas=193725 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[33096, 32542, 32768, 32701, 33096, 32066, 32779, 33096] avg=32768.0000 max=33096 max_rank=0 min=32066 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64142 max_load_reduction_pct=0.659639 moved_to_replicas=193725 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[65136, 32737, 24373, 32676, 528, 56848, 232, 49614] avg=32768.0000 max=65136 max_rank=0 min=232 min_rank=6 imbalance_ratio=1.987793 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[33096, 32542, 32768, 32701, 33096, 32066, 32779, 33096] avg=32768.0000 max=33096 max_rank=0 min=32066 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64142 max_load_reduction_pct=0.659639 moved_to_replicas=193725 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[33372, 1, 32768, 263, 97238, 189, 32779, 65534] avg=32768.0000 max=97238 max_rank=4 min=1 min_rank=1 imbalance_ratio=2.967468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[33372, 1, 32768, 263, 97238, 189, 32779, 65534] avg=32768.0000 max=97238 max_rank=4 min=1 min_rank=1 imbalance_ratio=2.967468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[33372, 1, 32768, 263, 97238, 189, 32779, 65534] avg=32768.0000 max=97238 max_rank=4 min=1 min_rank=1 imbalance_ratio=2.967468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32737, 32074, 32676, 33094, 33098, 32272, 33097] avg=32768.0000 max=33098 max_rank=5 min=32074 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32038 max_load_reduction_pct=0.491863 moved_to_replicas=142022 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[33372, 1, 32768, 263, 97238, 189, 32779, 65534] avg=32768.0000 max=97238 max_rank=4 min=1 min_rank=1 imbalance_ratio=2.967468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 35854, 32770, 32775, 5, 65530, 29590] avg=32768.0000 max=65530 max_rank=6 min=5 min_rank=5 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 35854, 32770, 32775, 5, 65530, 29590] avg=32768.0000 max=65530 max_rank=6 min=5 min_rank=5 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[33096, 32542, 32768, 32701, 33096, 32066, 32779, 33096] avg=32768.0000 max=33096 max_rank=0 min=32066 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64142 max_load_reduction_pct=0.659639 moved_to_replicas=193725 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[33096, 32542, 32768, 32701, 33096, 32066, 32779, 33096] avg=32768.0000 max=33096 max_rank=0 min=32066 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64142 max_load_reduction_pct=0.659639 moved_to_replicas=193725 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 35854, 32770, 32775, 5, 65530, 29590] avg=32768.0000 max=65530 max_rank=6 min=5 min_rank=5 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[33096, 32542, 32768, 32701, 33096, 32066, 32779, 33096] avg=32768.0000 max=33096 max_rank=0 min=32066 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64142 max_load_reduction_pct=0.659639 moved_to_replicas=193725 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[33096, 32542, 32768, 32701, 33096, 32066, 32779, 33096] avg=32768.0000 max=33096 max_rank=0 min=32066 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64142 max_load_reduction_pct=0.659639 moved_to_replicas=193725 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 33096, 32770, 32775, 32439, 33096, 32348] avg=32768.0000 max=33096 max_rank=2 min=32348 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=70302 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 33096, 32770, 32775, 32439, 33096, 32348] avg=32768.0000 max=33096 max_rank=2 min=32348 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=70302 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[33372, 1, 32768, 263, 97238, 189, 32779, 65534] avg=32768.0000 max=97238 max_rank=4 min=1 min_rank=1 imbalance_ratio=2.967468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 33096, 32770, 32775, 32439, 33096, 32348] avg=32768.0000 max=33096 max_rank=2 min=32348 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=70302 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 35854, 32770, 32775, 5, 65530, 29590] avg=32768.0000 max=65530 max_rank=6 min=5 min_rank=5 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 35854, 32770, 32775, 5, 65530, 29590] avg=32768.0000 max=65530 max_rank=6 min=5 min_rank=5 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 35854, 32770, 32775, 5, 65530, 29590] avg=32768.0000 max=65530 max_rank=6 min=5 min_rank=5 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[33096, 32542, 32768, 32701, 33096, 32066, 32779, 33096] avg=32768.0000 max=33096 max_rank=0 min=32066 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64142 max_load_reduction_pct=0.659639 moved_to_replicas=193725 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 35854, 32770, 32775, 5, 65530, 29590] avg=32768.0000 max=65530 max_rank=6 min=5 min_rank=5 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 33096, 32770, 32775, 32439, 33096, 32348] avg=32768.0000 max=33096 max_rank=2 min=32348 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=70302 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[32765, 17, 10, 32814, 33607, 95067, 2389, 65475] avg=32768.0000 max=95067 max_rank=5 min=10 min_rank=2 imbalance_ratio=2.901215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[32765, 17, 10, 32814, 33607, 95067, 2389, 65475] avg=32768.0000 max=95067 max_rank=5 min=10 min_rank=2 imbalance_ratio=2.901215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 33096, 32770, 32775, 32439, 33096, 32348] avg=32768.0000 max=33096 max_rank=2 min=32348 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=70302 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[32765, 17, 10, 32814, 33607, 95067, 2389, 65475] avg=32768.0000 max=95067 max_rank=5 min=10 min_rank=2 imbalance_ratio=2.901215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 33096, 32770, 32775, 32439, 33096, 32348] avg=32768.0000 max=33096 max_rank=2 min=32348 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=70302 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 33096, 32770, 32775, 32439, 33096, 32348] avg=32768.0000 max=33096 max_rank=2 min=32348 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=70302 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[32765, 31405, 32776, 32814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31405 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61971 max_load_reduction_pct=0.651867 moved_to_replicas=188752 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[32765, 31405, 32776, 32814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31405 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61971 max_load_reduction_pct=0.651867 moved_to_replicas=188752 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 35854, 32770, 32775, 5, 65530, 29590] avg=32768.0000 max=65530 max_rank=6 min=5 min_rank=5 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[32765, 31405, 32776, 32814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31405 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61971 max_load_reduction_pct=0.651867 moved_to_replicas=188752 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[32765, 17, 10, 32814, 33607, 95067, 2389, 65475] avg=32768.0000 max=95067 max_rank=5 min=10 min_rank=2 imbalance_ratio=2.901215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[32765, 17, 10, 32814, 33607, 95067, 2389, 65475] avg=32768.0000 max=95067 max_rank=5 min=10 min_rank=2 imbalance_ratio=2.901215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[32765, 17, 10, 32814, 33607, 95067, 2389, 65475] avg=32768.0000 max=95067 max_rank=5 min=10 min_rank=2 imbalance_ratio=2.901215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 33096, 32770, 32775, 32439, 33096, 32348] avg=32768.0000 max=33096 max_rank=2 min=32348 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=70302 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[32765, 17, 10, 32814, 33607, 95067, 2389, 65475] avg=32768.0000 max=95067 max_rank=5 min=10 min_rank=2 imbalance_ratio=2.901215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[32765, 31405, 32776, 32814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31405 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61971 max_load_reduction_pct=0.651867 moved_to_replicas=188752 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[34731, 3, 32767, 30941, 32780, 65376, 32777, 32769] avg=32768.0000 max=65376 max_rank=5 min=3 min_rank=1 imbalance_ratio=1.995117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[34731, 3, 32767, 30941, 32780, 65376, 32777, 32769] avg=32768.0000 max=65376 max_rank=5 min=3 min_rank=1 imbalance_ratio=1.995117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[34731, 3, 32767, 30941, 32780, 65376, 32777, 32769] avg=32768.0000 max=65376 max_rank=5 min=3 min_rank=1 imbalance_ratio=1.995117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[32765, 31405, 32776, 32814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31405 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61971 max_load_reduction_pct=0.651867 moved_to_replicas=188752 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[32765, 31405, 32776, 32814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31405 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61971 max_load_reduction_pct=0.651867 moved_to_replicas=188752 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[32765, 31405, 32776, 32814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31405 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61971 max_load_reduction_pct=0.651867 moved_to_replicas=188752 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32283, 32767, 32576, 32780, 33096, 32777, 32769] avg=32768.0000 max=33096 max_rank=0 min=32283 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32280 max_load_reduction_pct=0.493759 moved_to_replicas=66144 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[32765, 17, 10, 32814, 33607, 95067, 2389, 65475] avg=32768.0000 max=95067 max_rank=5 min=10 min_rank=2 imbalance_ratio=2.901215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32283, 32767, 32576, 32780, 33096, 32777, 32769] avg=32768.0000 max=33096 max_rank=0 min=32283 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32280 max_load_reduction_pct=0.493759 moved_to_replicas=66144 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32283, 32767, 32576, 32780, 33096, 32777, 32769] avg=32768.0000 max=33096 max_rank=0 min=32283 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32280 max_load_reduction_pct=0.493759 moved_to_replicas=66144 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[34731, 3, 32767, 30941, 32780, 65376, 32777, 32769] avg=32768.0000 max=65376 max_rank=5 min=3 min_rank=1 imbalance_ratio=1.995117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[34731, 3, 32767, 30941, 32780, 65376, 32777, 32769] avg=32768.0000 max=65376 max_rank=5 min=3 min_rank=1 imbalance_ratio=1.995117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[34731, 3, 32767, 30941, 32780, 65376, 32777, 32769] avg=32768.0000 max=65376 max_rank=5 min=3 min_rank=1 imbalance_ratio=1.995117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[32765, 31405, 32776, 32814, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31405 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61971 max_load_reduction_pct=0.651867 moved_to_replicas=188752 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[34731, 3, 32767, 30941, 32780, 65376, 32777, 32769] avg=32768.0000 max=65376 max_rank=5 min=3 min_rank=1 imbalance_ratio=1.995117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32283, 32767, 32576, 32780, 33096, 32777, 32769] avg=32768.0000 max=33096 max_rank=0 min=32283 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32280 max_load_reduction_pct=0.493759 moved_to_replicas=66144 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[33698, 64794, 2, 32769, 32838, 1, 32765, 65277] avg=32768.0000 max=65277 max_rank=7 min=1 min_rank=5 imbalance_ratio=1.992096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[33698, 64794, 2, 32769, 32838, 1, 32765, 65277] avg=32768.0000 max=65277 max_rank=7 min=1 min_rank=5 imbalance_ratio=1.992096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[33698, 64794, 2, 32769, 32838, 1, 32765, 65277] avg=32768.0000 max=65277 max_rank=7 min=1 min_rank=5 imbalance_ratio=1.992096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32283, 32767, 32576, 32780, 33096, 32777, 32769] avg=32768.0000 max=33096 max_rank=0 min=32283 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32280 max_load_reduction_pct=0.493759 moved_to_replicas=66144 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32283, 32767, 32576, 32780, 33096, 32777, 32769] avg=32768.0000 max=33096 max_rank=0 min=32283 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32280 max_load_reduction_pct=0.493759 moved_to_replicas=66144 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32283, 32767, 32576, 32780, 33096, 32777, 32769] avg=32768.0000 max=33096 max_rank=0 min=32283 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32280 max_load_reduction_pct=0.493759 moved_to_replicas=66144 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[33096, 33096, 32302, 32769, 32838, 32182, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32182 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.492991 moved_to_replicas=129175 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[33096, 33096, 32302, 32769, 32838, 32182, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32182 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.492991 moved_to_replicas=129175 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[34731, 3, 32767, 30941, 32780, 65376, 32777, 32769] avg=32768.0000 max=65376 max_rank=5 min=3 min_rank=1 imbalance_ratio=1.995117 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[33096, 33096, 32302, 32769, 32838, 32182, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32182 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.492991 moved_to_replicas=129175 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[33698, 64794, 2, 32769, 32838, 1, 32765, 65277] avg=32768.0000 max=65277 max_rank=7 min=1 min_rank=5 imbalance_ratio=1.992096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[33698, 64794, 2, 32769, 32838, 1, 32765, 65277] avg=32768.0000 max=65277 max_rank=7 min=1 min_rank=5 imbalance_ratio=1.992096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[33698, 64794, 2, 32769, 32838, 1, 32765, 65277] avg=32768.0000 max=65277 max_rank=7 min=1 min_rank=5 imbalance_ratio=1.992096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32283, 32767, 32576, 32780, 33096, 32777, 32769] avg=32768.0000 max=33096 max_rank=0 min=32283 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32280 max_load_reduction_pct=0.493759 moved_to_replicas=66144 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[33698, 64794, 2, 32769, 32838, 1, 32765, 65277] avg=32768.0000 max=65277 max_rank=7 min=1 min_rank=5 imbalance_ratio=1.992096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[33096, 33096, 32302, 32769, 32838, 32182, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32182 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.492991 moved_to_replicas=129175 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[32766, 2, 98260, 65536, 2, 3, 32764, 32811] avg=32768.0000 max=98260 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[32766, 2, 98260, 65536, 2, 3, 32764, 32811] avg=32768.0000 max=98260 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[33096, 33096, 32302, 32769, 32838, 32182, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32182 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.492991 moved_to_replicas=129175 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[33096, 33096, 32302, 32769, 32838, 32182, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32182 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.492991 moved_to_replicas=129175 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[32766, 2, 98260, 65536, 2, 3, 32764, 32811] avg=32768.0000 max=98260 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[33096, 33096, 32302, 32769, 32838, 32182, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32182 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.492991 moved_to_replicas=129175 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[32766, 32768, 33096, 33096, 32400, 32443, 32764, 32811] avg=32768.0000 max=33096 max_rank=2 min=32400 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=195009 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[32766, 32768, 33096, 33096, 32400, 32443, 32764, 32811] avg=32768.0000 max=33096 max_rank=2 min=32400 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=195009 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[33698, 64794, 2, 32769, 32838, 1, 32765, 65277] avg=32768.0000 max=65277 max_rank=7 min=1 min_rank=5 imbalance_ratio=1.992096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[32766, 32768, 33096, 33096, 32400, 32443, 32764, 32811] avg=32768.0000 max=33096 max_rank=2 min=32400 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=195009 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[32766, 2, 98260, 65536, 2, 3, 32764, 32811] avg=32768.0000 max=98260 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[32766, 2, 98260, 65536, 2, 3, 32764, 32811] avg=32768.0000 max=98260 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[32766, 2, 98260, 65536, 2, 3, 32764, 32811] avg=32768.0000 max=98260 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[33096, 33096, 32302, 32769, 32838, 32182, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32182 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.492991 moved_to_replicas=129175 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[32766, 2, 98260, 65536, 2, 3, 32764, 32811] avg=32768.0000 max=98260 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[32766, 32768, 33096, 33096, 32400, 32443, 32764, 32811] avg=32768.0000 max=33096 max_rank=2 min=32400 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=195009 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[32787, 54, 32829, 65528, 65436, 2, 4, 65504] avg=32768.0000 max=65528 max_rank=3 min=2 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[32787, 54, 32829, 65528, 65436, 2, 4, 65504] avg=32768.0000 max=65528 max_rank=3 min=2 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[32766, 32768, 33096, 33096, 32400, 32443, 32764, 32811] avg=32768.0000 max=33096 max_rank=2 min=32400 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=195009 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[32766, 32768, 33096, 33096, 32400, 32443, 32764, 32811] avg=32768.0000 max=33096 max_rank=2 min=32400 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=195009 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[32787, 54, 32829, 65528, 65436, 2, 4, 65504] avg=32768.0000 max=65528 max_rank=3 min=2 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[32766, 32768, 33096, 33096, 32400, 32443, 32764, 32811] avg=32768.0000 max=33096 max_rank=2 min=32400 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=195009 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[32787, 32394, 32829, 33096, 33096, 32434, 32412, 33096] avg=32768.0000 max=33096 max_rank=3 min=32394 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194211 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[32766, 2, 98260, 65536, 2, 3, 32764, 32811] avg=32768.0000 max=98260 max_rank=2 min=2 min_rank=1 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[32787, 32394, 32829, 33096, 33096, 32434, 32412, 33096] avg=32768.0000 max=33096 max_rank=3 min=32394 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194211 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[32787, 54, 32829, 65528, 65436, 2, 4, 65504] avg=32768.0000 max=65528 max_rank=3 min=2 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[32787, 32394, 32829, 33096, 33096, 32434, 32412, 33096] avg=32768.0000 max=33096 max_rank=3 min=32394 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194211 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[32787, 54, 32829, 65528, 65436, 2, 4, 65504] avg=32768.0000 max=65528 max_rank=3 min=2 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[32787, 54, 32829, 65528, 65436, 2, 4, 65504] avg=32768.0000 max=65528 max_rank=3 min=2 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[32766, 32768, 33096, 33096, 32400, 32443, 32764, 32811] avg=32768.0000 max=33096 max_rank=2 min=32400 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=195009 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[32787, 32394, 32829, 33096, 33096, 32434, 32412, 33096] avg=32768.0000 max=33096 max_rank=3 min=32394 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194211 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[32787, 54, 32829, 65528, 65436, 2, 4, 65504] avg=32768.0000 max=65528 max_rank=3 min=2 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[31412, 32772, 103, 65532, 32930, 2, 65515, 33878] avg=32768.0000 max=65532 max_rank=3 min=2 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[32787, 32394, 32829, 33096, 33096, 32434, 32412, 33096] avg=32768.0000 max=33096 max_rank=3 min=32394 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194211 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[31412, 32772, 103, 65532, 32930, 2, 65515, 33878] avg=32768.0000 max=65532 max_rank=3 min=2 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[32787, 32394, 32829, 33096, 33096, 32434, 32412, 33096] avg=32768.0000 max=33096 max_rank=3 min=32394 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194211 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[31412, 32772, 103, 65532, 32930, 2, 65515, 33878] avg=32768.0000 max=65532 max_rank=3 min=2 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[32787, 32394, 32829, 33096, 33096, 32434, 32412, 33096] avg=32768.0000 max=33096 max_rank=3 min=32394 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194211 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[32194, 32772, 32522, 33096, 32930, 32438, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32194 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=132628 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[32787, 54, 32829, 65528, 65436, 2, 4, 65504] avg=32768.0000 max=65528 max_rank=3 min=2 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[32194, 32772, 32522, 33096, 32930, 32438, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32194 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=132628 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[31412, 32772, 103, 65532, 32930, 2, 65515, 33878] avg=32768.0000 max=65532 max_rank=3 min=2 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[32194, 32772, 32522, 33096, 32930, 32438, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32194 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=132628 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[31412, 32772, 103, 65532, 32930, 2, 65515, 33878] avg=32768.0000 max=65532 max_rank=3 min=2 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[31412, 32772, 103, 65532, 32930, 2, 65515, 33878] avg=32768.0000 max=65532 max_rank=3 min=2 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[32787, 32394, 32829, 33096, 33096, 32434, 32412, 33096] avg=32768.0000 max=33096 max_rank=3 min=32394 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194211 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[32194, 32772, 32522, 33096, 32930, 32438, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32194 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=132628 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[0, 32769, 5, 65429, 3, 65531, 32769, 65638] avg=32768.0000 max=65638 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[31412, 32772, 103, 65532, 32930, 2, 65515, 33878] avg=32768.0000 max=65532 max_rank=3 min=2 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[32194, 32772, 32522, 33096, 32930, 32438, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32194 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=132628 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[0, 32769, 5, 65429, 3, 65531, 32769, 65638] avg=32768.0000 max=65638 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[32194, 32772, 32522, 33096, 32930, 32438, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32194 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=132628 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[0, 32769, 5, 65429, 3, 65531, 32769, 65638] avg=32768.0000 max=65638 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[32542, 32769, 32338, 33096, 32438, 33096, 32769, 33096] avg=32768.0000 max=33096 max_rank=3 min=32338 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32542 max_load_reduction_pct=0.495780 moved_to_replicas=194259 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[32194, 32772, 32522, 33096, 32930, 32438, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32194 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=132628 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[31412, 32772, 103, 65532, 32930, 2, 65515, 33878] avg=32768.0000 max=65532 max_rank=3 min=2 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[0, 32769, 5, 65429, 3, 65531, 32769, 65638] avg=32768.0000 max=65638 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[32542, 32769, 32338, 33096, 32438, 33096, 32769, 33096] avg=32768.0000 max=33096 max_rank=3 min=32338 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32542 max_load_reduction_pct=0.495780 moved_to_replicas=194259 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[32542, 32769, 32338, 33096, 32438, 33096, 32769, 33096] avg=32768.0000 max=33096 max_rank=3 min=32338 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32542 max_load_reduction_pct=0.495780 moved_to_replicas=194259 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[0, 32769, 5, 65429, 3, 65531, 32769, 65638] avg=32768.0000 max=65638 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[0, 32769, 5, 65429, 3, 65531, 32769, 65638] avg=32768.0000 max=65638 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[32194, 32772, 32522, 33096, 32930, 32438, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32194 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=132628 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[3189, 32765, 6, 32762, 98301, 32767, 5, 62349] avg=32768.0000 max=98301 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[32542, 32769, 32338, 33096, 32438, 33096, 32769, 33096] avg=32768.0000 max=33096 max_rank=3 min=32338 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32542 max_load_reduction_pct=0.495780 moved_to_replicas=194259 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[0, 32769, 5, 65429, 3, 65531, 32769, 65638] avg=32768.0000 max=65638 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[32542, 32769, 32338, 33096, 32438, 33096, 32769, 33096] avg=32768.0000 max=33096 max_rank=3 min=32338 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32542 max_load_reduction_pct=0.495780 moved_to_replicas=194259 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[32542, 32769, 32338, 33096, 32438, 33096, 32769, 33096] avg=32768.0000 max=33096 max_rank=3 min=32338 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32542 max_load_reduction_pct=0.495780 moved_to_replicas=194259 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[3189, 32765, 6, 32762, 98301, 32767, 5, 62349] avg=32768.0000 max=98301 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[3189, 32765, 6, 32762, 98301, 32767, 5, 62349] avg=32768.0000 max=98301 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[32442, 32765, 32445, 32762, 33096, 32767, 32771, 33096] avg=32768.0000 max=33096 max_rank=4 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=188612 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[32542, 32769, 32338, 33096, 32438, 33096, 32769, 33096] avg=32768.0000 max=33096 max_rank=3 min=32338 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32542 max_load_reduction_pct=0.495780 moved_to_replicas=194259 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[0, 32769, 5, 65429, 3, 65531, 32769, 65638] avg=32768.0000 max=65638 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[32442, 32765, 32445, 32762, 33096, 32767, 32771, 33096] avg=32768.0000 max=33096 max_rank=4 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=188612 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[3189, 32765, 6, 32762, 98301, 32767, 5, 62349] avg=32768.0000 max=98301 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[3189, 32765, 6, 32762, 98301, 32767, 5, 62349] avg=32768.0000 max=98301 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[32442, 32765, 32445, 32762, 33096, 32767, 32771, 33096] avg=32768.0000 max=33096 max_rank=4 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=188612 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[3189, 32765, 6, 32762, 98301, 32767, 5, 62349] avg=32768.0000 max=98301 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[32542, 32769, 32338, 33096, 32438, 33096, 32769, 33096] avg=32768.0000 max=33096 max_rank=3 min=32338 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32542 max_load_reduction_pct=0.495780 moved_to_replicas=194259 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[32765, 65536, 32764, 3, 32766, 32764, 32, 65514] avg=32768.0000 max=65536 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[32442, 32765, 32445, 32762, 33096, 32767, 32771, 33096] avg=32768.0000 max=33096 max_rank=4 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=188612 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[3189, 32765, 6, 32762, 98301, 32767, 5, 62349] avg=32768.0000 max=98301 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[32442, 32765, 32445, 32762, 33096, 32767, 32771, 33096] avg=32768.0000 max=33096 max_rank=4 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=188612 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[32442, 32765, 32445, 32762, 33096, 32767, 32771, 33096] avg=32768.0000 max=33096 max_rank=4 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=188612 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[32765, 65536, 32764, 3, 32766, 32764, 32, 65514] avg=32768.0000 max=65536 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[32765, 65536, 32764, 3, 32766, 32764, 32, 65514] avg=32768.0000 max=65536 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[32765, 33096, 32764, 32443, 32766, 32764, 32450, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=129574 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[3189, 32765, 6, 32762, 98301, 32767, 5, 62349] avg=32768.0000 max=98301 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[32442, 32765, 32445, 32762, 33096, 32767, 32771, 33096] avg=32768.0000 max=33096 max_rank=4 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=188612 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[32765, 33096, 32764, 32443, 32766, 32764, 32450, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=129574 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[32765, 65536, 32764, 3, 32766, 32764, 32, 65514] avg=32768.0000 max=65536 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[32765, 33096, 32764, 32443, 32766, 32764, 32450, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=129574 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[32765, 65536, 32764, 3, 32766, 32764, 32, 65514] avg=32768.0000 max=65536 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[32765, 65536, 32764, 3, 32766, 32764, 32, 65514] avg=32768.0000 max=65536 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[32442, 32765, 32445, 32762, 33096, 32767, 32771, 33096] avg=32768.0000 max=33096 max_rank=4 min=32442 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=188612 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[32765, 33096, 32764, 32443, 32766, 32764, 32450, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=129574 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[32765, 33096, 32764, 32443, 32766, 32764, 32450, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=129574 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[32765, 65536, 32764, 3, 32766, 32764, 32, 65514] avg=32768.0000 max=65536 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[32765, 33096, 32764, 32443, 32766, 32764, 32450, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=129574 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[32765, 33096, 32764, 32443, 32766, 32764, 32450, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=129574 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[32765, 65536, 32764, 3, 32766, 32764, 32, 65514] avg=32768.0000 max=65536 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[32765, 33096, 32764, 32443, 32766, 32764, 32450, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=129574 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=32 total_expected_recv=1046753 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56832 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=32 total_expected_recv=1045467 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=58368 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=32 total_expected_recv=1041822 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=61952 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=32 total_expected_recv=1050925 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=53504 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=32 total_expected_recv=1048068 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56704 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=32 total_expected_recv=1052231 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=51712 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=32 total_expected_recv=1053915 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=51456 peak_demand=33099 required_capacity=1.010101 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=32 total_expected_recv=1049427 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=54144 peak_demand=33096 required_capacity=1.010010 + [2026-08-30 05:11:42] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 529.6 | throughput per GPU (TFLOP/s/GPU): 629.5 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.039298E+01 | loss scale: 1.0 | grad norm: 16.294 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30472, 33095, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30472, 33095, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30472, 33095, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=34816 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30472, 33095, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=34816 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30472, 33095, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30472, 33095, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30472, 33095, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30472, 33095, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=34816 expected_recv=60944 actual_valid_recv=61696 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=34816 expected_recv=66190 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=61696 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=67200 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=34816 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[33096, 32172, 33096, 32621, 32779, 32764, 32732, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32172, 33096, 32621, 32779, 32764, 32732, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=34816 expected_recv=64844 expected_overflow=0 exact_recv_per_rank=[33096, 32172, 33096, 32621, 32779, 32764, 32732, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=34816 expected_recv=65530 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=34816 expected_recv=65235 expected_overflow=0 exact_recv_per_rank=[33096, 32172, 33096, 32621, 32779, 32764, 32732, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32172, 33096, 32621, 32779, 32764, 32732, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=34816 expected_recv=64844 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=34816 expected_recv=65146 expected_overflow=0 exact_recv_per_rank=[33096, 32172, 33096, 32621, 32779, 32764, 32732, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=34816 expected_recv=65654 expected_overflow=0 exact_recv_per_rank=[33096, 32172, 33096, 32621, 32779, 32764, 32732, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=34816 expected_recv=65495 expected_overflow=0 exact_recv_per_rank=[33096, 32172, 33096, 32621, 32779, 32764, 32732, 32884] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=34816 expected_recv=65235 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=34816 expected_recv=65146 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=34816 expected_recv=65654 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=34816 expected_recv=65495 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=34816 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[32778, 32779, 33096, 31789, 32416, 33094, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=34816 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[32778, 32779, 33096, 31789, 32416, 33094, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=34816 expected_recv=66190 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=34816 expected_recv=65640 expected_overflow=0 exact_recv_per_rank=[32778, 32779, 33096, 31789, 32416, 33094, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=34816 expected_recv=65533 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32778, 32779, 33096, 31789, 32416, 33094, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=34816 expected_recv=63487 expected_overflow=0 exact_recv_per_rank=[32778, 32779, 33096, 31789, 32416, 33094, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32778, 32779, 33096, 31789, 32416, 33094, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32778, 32779, 33096, 31789, 32416, 33094, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=34816 expected_recv=64862 expected_overflow=0 exact_recv_per_rank=[32778, 32779, 33096, 31789, 32416, 33094, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=34816 expected_recv=65640 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=34816 expected_recv=63487 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=34816 expected_recv=64862 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=64512 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=67200 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=34816 expected_recv=64872 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32788, 32407, 32782, 32451, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=34816 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32788, 32407, 32782, 32451, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=34816 expected_recv=64872 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32788, 32407, 32782, 32451, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=34816 expected_recv=65533 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=34816 expected_recv=65582 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32788, 32407, 32782, 32451, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=34816 expected_recv=64838 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32788, 32407, 32782, 32451, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=34816 expected_recv=65507 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32788, 32407, 32782, 32451, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=34816 expected_recv=65572 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32788, 32407, 32782, 32451, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=34816 expected_recv=64838 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32788, 32407, 32782, 32451, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=34816 expected_recv=65582 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=34816 expected_recv=65507 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=34816 expected_recv=65572 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32754, 32276, 32376, 33020, 33096, 33096, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32754, 32276, 32376, 33020, 33096, 33096, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=34816 expected_recv=65392 expected_overflow=0 exact_recv_per_rank=[33096, 32754, 32276, 32376, 33020, 33096, 33096, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=34816 expected_recv=64766 expected_overflow=0 exact_recv_per_rank=[33096, 32754, 32276, 32376, 33020, 33096, 33096, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=34816 expected_recv=64568 expected_overflow=0 exact_recv_per_rank=[33096, 32754, 32276, 32376, 33020, 33096, 33096, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=34816 expected_recv=65392 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32754, 32276, 32376, 33020, 33096, 33096, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=34816 expected_recv=66116 expected_overflow=0 exact_recv_per_rank=[33096, 32754, 32276, 32376, 33020, 33096, 33096, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=34816 expected_recv=64766 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=34816 expected_recv=64568 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=34816 expected_recv=64870 expected_overflow=0 exact_recv_per_rank=[33096, 32754, 32276, 32376, 33020, 33096, 33096, 32430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=34816 expected_recv=66116 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 32770, 32751, 32103, 33096, 32458, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 32770, 32751, 32103, 33096, 32458, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=34816 expected_recv=64870 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=34816 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 32770, 32751, 32103, 33096, 32458, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=34816 expected_recv=65353 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 32770, 32751, 32103, 33096, 32458, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=34816 expected_recv=64387 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 32770, 32751, 32103, 33096, 32458, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=34816 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 32770, 32751, 32103, 33096, 32458, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=34816 expected_recv=65537 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=34816 expected_recv=64898 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 32770, 32751, 32103, 33096, 32458, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=34816 expected_recv=64276 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 33096, 32506, 33096, 32194, 32624, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 32770, 32751, 32103, 33096, 32458, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=34816 expected_recv=65353 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=34816 expected_recv=64387 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=34816 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 33096, 32506, 33096, 32194, 32624, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=34816 expected_recv=65537 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=34816 expected_recv=64898 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=34816 expected_recv=64276 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=34816 expected_recv=65537 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 33096, 32506, 33096, 32194, 32624, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=34816 expected_recv=65228 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 33096, 32506, 33096, 32194, 32624, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 33096, 32506, 33096, 32194, 32624, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 33096, 32506, 33096, 32194, 32624, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32367, 32816, 32769, 32767, 32769, 33096, 33096, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=34816 expected_recv=65228 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=34816 expected_recv=65146 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 33096, 32506, 33096, 32194, 32624, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=34816 expected_recv=64819 expected_overflow=0 exact_recv_per_rank=[32367, 32816, 32769, 32767, 32769, 33096, 33096, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=34816 expected_recv=65525 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 33096, 32506, 33096, 32194, 32624, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=34816 expected_recv=65146 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=34816 expected_recv=64819 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=34816 expected_recv=65525 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=34816 expected_recv=65659 expected_overflow=0 exact_recv_per_rank=[32367, 32816, 32769, 32767, 32769, 33096, 33096, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=34816 expected_recv=65513 expected_overflow=0 exact_recv_per_rank=[32367, 32816, 32769, 32767, 32769, 33096, 33096, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=34816 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[32367, 32816, 32769, 32767, 32769, 33096, 33096, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=34816 expected_recv=65659 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=34816 expected_recv=65513 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=34816 expected_recv=64873 expected_overflow=0 exact_recv_per_rank=[33096, 32128, 32796, 32394, 32774, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=34816 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[32367, 32816, 32769, 32767, 32769, 33096, 33096, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=34816 expected_recv=65533 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32367, 32816, 32769, 32767, 32769, 33096, 33096, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32128, 32796, 32394, 32774, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=34816 expected_recv=64836 expected_overflow=0 exact_recv_per_rank=[32367, 32816, 32769, 32767, 32769, 33096, 33096, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=34816 expected_recv=64873 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=34816 expected_recv=65544 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=34816 expected_recv=64836 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=34816 expected_recv=64550 expected_overflow=0 exact_recv_per_rank=[33096, 32128, 32796, 32394, 32774, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=34816 expected_recv=65171 expected_overflow=0 exact_recv_per_rank=[33096, 32128, 32796, 32394, 32774, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=34816 expected_recv=64550 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=34816 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[33096, 32128, 32796, 32394, 32774, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=34816 expected_recv=65171 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=34816 expected_recv=65549 expected_overflow=0 exact_recv_per_rank=[32841, 33096, 32402, 32771, 32714, 32774, 32755, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=34816 expected_recv=65546 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=34816 expected_recv=65572 expected_overflow=0 exact_recv_per_rank=[33096, 32128, 32796, 32394, 32774, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=34816 expected_recv=65678 expected_overflow=0 exact_recv_per_rank=[32841, 33096, 32402, 32771, 32714, 32774, 32755, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32128, 32796, 32394, 32774, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32128, 32796, 32394, 32774, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=34816 expected_recv=65549 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=34816 expected_recv=65572 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=34816 expected_recv=65678 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32841, 33096, 32402, 32771, 32714, 32774, 32755, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=34816 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[32841, 33096, 32402, 32771, 32714, 32774, 32755, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=34816 expected_recv=65485 expected_overflow=0 exact_recv_per_rank=[32841, 33096, 32402, 32771, 32714, 32774, 32755, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=34816 expected_recv=65541 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32388, 31701, 33096, 32894, 33096, 33096, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=34816 expected_recv=65485 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=34816 expected_recv=64770 expected_overflow=0 exact_recv_per_rank=[32841, 33096, 32402, 32771, 32714, 32774, 32755, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=34816 expected_recv=64780 expected_overflow=0 exact_recv_per_rank=[32388, 31701, 33096, 32894, 33096, 33096, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=34816 expected_recv=65511 expected_overflow=0 exact_recv_per_rank=[32841, 33096, 32402, 32771, 32714, 32774, 32755, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=34816 expected_recv=64770 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=34816 expected_recv=65562 expected_overflow=0 exact_recv_per_rank=[32841, 33096, 32402, 32771, 32714, 32774, 32755, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=34816 expected_recv=64780 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=34816 expected_recv=65511 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=34816 expected_recv=65562 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=34816 expected_recv=63402 expected_overflow=0 exact_recv_per_rank=[32388, 31701, 33096, 32894, 33096, 33096, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=34816 expected_recv=65769 expected_overflow=0 exact_recv_per_rank=[32388, 31701, 33096, 32894, 33096, 33096, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=34816 expected_recv=63402 actual_valid_recv=63872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32388, 31701, 33096, 32894, 33096, 33096, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=34816 expected_recv=65554 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32774, 32462, 32461, 32775, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=34816 expected_recv=65769 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=63872 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32388, 31701, 33096, 32894, 33096, 33096, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=34816 expected_recv=65554 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32774, 32462, 32461, 32775, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=34816 expected_recv=65569 expected_overflow=0 exact_recv_per_rank=[32388, 31701, 33096, 32894, 33096, 33096, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32388, 31701, 33096, 32894, 33096, 33096, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=34816 expected_recv=65569 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=34816 expected_recv=65369 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32774, 32462, 32461, 32775, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=34816 expected_recv=64985 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32774, 32462, 32461, 32775, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=34816 expected_recv=65369 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=34816 expected_recv=64896 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32774, 32462, 32461, 32775, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=34816 expected_recv=64881 expected_overflow=0 exact_recv_per_rank=[32786, 33096, 33096, 32754, 33096, 31785, 32765, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=34816 expected_recv=64985 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=34816 expected_recv=64896 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=34816 expected_recv=64881 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=34816 expected_recv=65562 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32774, 32462, 32461, 32775, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=34816 expected_recv=65562 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=34816 expected_recv=65585 expected_overflow=0 exact_recv_per_rank=[32786, 33096, 33096, 32754, 33096, 31785, 32765, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=34816 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32774, 32462, 32461, 32775, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32709, 32774, 32462, 32461, 32775, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=34816 expected_recv=65585 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=34816 expected_recv=65538 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32786, 33096, 33096, 32754, 33096, 31785, 32765, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=34816 expected_recv=65500 expected_overflow=0 exact_recv_per_rank=[32786, 33096, 33096, 32754, 33096, 31785, 32765, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=34816 expected_recv=65683 expected_overflow=0 exact_recv_per_rank=[32394, 32780, 33096, 32766, 32443, 32793, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=34816 expected_recv=65500 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32786, 33096, 33096, 32754, 33096, 31785, 32765, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=34816 expected_recv=65683 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=34816 expected_recv=64865 expected_overflow=0 exact_recv_per_rank=[32786, 33096, 33096, 32754, 33096, 31785, 32765, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=34816 expected_recv=64865 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=34816 expected_recv=64714 expected_overflow=0 exact_recv_per_rank=[32394, 32780, 33096, 32766, 32443, 32793, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=34816 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[32786, 33096, 33096, 32754, 33096, 31785, 32765, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=34816 expected_recv=64714 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=34816 expected_recv=65527 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=34816 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[32786, 33096, 33096, 32754, 33096, 31785, 32765, 32766] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=34816 expected_recv=65564 expected_overflow=0 exact_recv_per_rank=[32394, 32780, 33096, 32766, 32443, 32793, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=34816 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[32394, 32780, 33096, 32766, 32443, 32793, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=34816 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[32766, 32435, 32788, 32758, 33096, 32779, 33096, 32426] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=34816 expected_recv=64891 expected_overflow=0 exact_recv_per_rank=[32394, 32780, 33096, 32766, 32443, 32793, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=34816 expected_recv=65545 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=34816 expected_recv=65564 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=34816 expected_recv=65529 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=34816 expected_recv=65541 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=34816 expected_recv=64891 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32394, 32780, 33096, 32766, 32443, 32793, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=34816 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[32766, 32435, 32788, 32758, 33096, 32779, 33096, 32426] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=34816 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[32394, 32780, 33096, 32766, 32443, 32793, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=34816 expected_recv=65529 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=34816 expected_recv=65523 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32394, 32780, 33096, 32766, 32443, 32793, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=34816 expected_recv=64881 expected_overflow=0 exact_recv_per_rank=[32766, 32435, 32788, 32758, 33096, 32779, 33096, 32426] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=34816 expected_recv=64867 expected_overflow=0 exact_recv_per_rank=[33096, 32800, 32417, 32437, 32768, 32434, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=34816 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[32766, 32435, 32788, 32758, 33096, 32779, 33096, 32426] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32766, 32435, 32788, 32758, 33096, 32779, 33096, 32426] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=34816 expected_recv=64881 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=34816 expected_recv=64867 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=34816 expected_recv=65523 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=34816 expected_recv=65584 expected_overflow=0 exact_recv_per_rank=[32766, 32435, 32788, 32758, 33096, 32779, 33096, 32426] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32800, 32417, 32437, 32768, 32434, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=34816 expected_recv=65584 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32766, 32435, 32788, 32758, 33096, 32779, 33096, 32426] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=34816 expected_recv=64846 expected_overflow=0 exact_recv_per_rank=[32766, 32435, 32788, 32758, 33096, 32779, 33096, 32426] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=34816 expected_recv=65599 expected_overflow=0 exact_recv_per_rank=[33096, 32800, 32417, 32437, 32768, 32434, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=34816 expected_recv=64884 expected_overflow=0 exact_recv_per_rank=[33096, 32800, 32417, 32437, 32768, 32434, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=34816 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[33096, 32800, 32417, 32437, 32768, 32434, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[25081, 18817, 29607, 47010, 49430, 29168, 20930, 42101] avg=32768.0000 max=49430 max_rank=4 min=18817 min_rank=1 imbalance_ratio=1.508484 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=34816 expected_recv=64846 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=34816 expected_recv=65599 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=34816 expected_recv=64884 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=34816 expected_recv=64826 expected_overflow=0 exact_recv_per_rank=[33096, 32800, 32417, 32437, 32768, 32434, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=34816 expected_recv=65536 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=34816 expected_recv=64826 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[25081, 18817, 29607, 47010, 49430, 29168, 20930, 42101] avg=32768.0000 max=49430 max_rank=4 min=18817 min_rank=1 imbalance_ratio=1.508484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 30472, 33095, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16333 max_load_reduction_pct=0.330427 moved_to_replicas=76231 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32800, 32417, 32437, 32768, 32434, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32800, 32417, 32437, 32768, 32434, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 30472, 33095, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16333 max_load_reduction_pct=0.330427 moved_to_replicas=76231 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[25081, 18817, 29607, 47010, 49430, 29168, 20930, 42101] avg=32768.0000 max=49430 max_rank=4 min=18817 min_rank=1 imbalance_ratio=1.508484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[25081, 18817, 29607, 47010, 49430, 29168, 20930, 42101] avg=32768.0000 max=49430 max_rank=4 min=18817 min_rank=1 imbalance_ratio=1.508484 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[25081, 18817, 29607, 47010, 49430, 29168, 20930, 42101] avg=32768.0000 max=49430 max_rank=4 min=18817 min_rank=1 imbalance_ratio=1.508484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[38570, 31365, 28429, 32621, 32779, 32764, 32732, 32884] avg=32768.0000 max=38570 max_rank=0 min=28429 min_rank=2 imbalance_ratio=1.177063 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[25081, 18817, 29607, 47010, 49430, 29168, 20930, 42101] avg=32768.0000 max=49430 max_rank=4 min=18817 min_rank=1 imbalance_ratio=1.508484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 30472, 33095, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16333 max_load_reduction_pct=0.330427 moved_to_replicas=76231 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 30472, 33095, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16333 max_load_reduction_pct=0.330427 moved_to_replicas=76231 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[38570, 31365, 28429, 32621, 32779, 32764, 32732, 32884] avg=32768.0000 max=38570 max_rank=0 min=28429 min_rank=2 imbalance_ratio=1.177063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32172, 33096, 32621, 32779, 32764, 32732, 32884] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=5474 max_load_reduction_pct=0.141924 moved_to_replicas=8656 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 30472, 33095, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16333 max_load_reduction_pct=0.330427 moved_to_replicas=76231 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[25081, 18817, 29607, 47010, 49430, 29168, 20930, 42101] avg=32768.0000 max=49430 max_rank=4 min=18817 min_rank=1 imbalance_ratio=1.508484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 30472, 33095, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16333 max_load_reduction_pct=0.330427 moved_to_replicas=76231 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[25081, 18817, 29607, 47010, 49430, 29168, 20930, 42101] avg=32768.0000 max=49430 max_rank=4 min=18817 min_rank=1 imbalance_ratio=1.508484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32172, 33096, 32621, 32779, 32764, 32732, 32884] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=5474 max_load_reduction_pct=0.141924 moved_to_replicas=8656 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[38570, 31365, 28429, 32621, 32779, 32764, 32732, 32884] avg=32768.0000 max=38570 max_rank=0 min=28429 min_rank=2 imbalance_ratio=1.177063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 30472, 33095, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16333 max_load_reduction_pct=0.330427 moved_to_replicas=76231 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[38570, 31365, 28429, 32621, 32779, 32764, 32732, 32884] avg=32768.0000 max=38570 max_rank=0 min=28429 min_rank=2 imbalance_ratio=1.177063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[32778, 32779, 55560, 23492, 2, 37502, 14521, 65510] avg=32768.0000 max=65510 max_rank=7 min=2 min_rank=4 imbalance_ratio=1.999207 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[38570, 31365, 28429, 32621, 32779, 32764, 32732, 32884] avg=32768.0000 max=38570 max_rank=0 min=28429 min_rank=2 imbalance_ratio=1.177063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 30472, 33095, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16333 max_load_reduction_pct=0.330427 moved_to_replicas=76231 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[38570, 31365, 28429, 32621, 32779, 32764, 32732, 32884] avg=32768.0000 max=38570 max_rank=0 min=28429 min_rank=2 imbalance_ratio=1.177063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32172, 33096, 32621, 32779, 32764, 32732, 32884] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=5474 max_load_reduction_pct=0.141924 moved_to_replicas=8656 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[32778, 32779, 55560, 23492, 2, 37502, 14521, 65510] avg=32768.0000 max=65510 max_rank=7 min=2 min_rank=4 imbalance_ratio=1.999207 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[32778, 32779, 33096, 31789, 32416, 33094, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31789 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32414 max_load_reduction_pct=0.494795 moved_to_replicas=122025 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32172, 33096, 32621, 32779, 32764, 32732, 32884] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=5474 max_load_reduction_pct=0.141924 moved_to_replicas=8656 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32172, 33096, 32621, 32779, 32764, 32732, 32884] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=5474 max_load_reduction_pct=0.141924 moved_to_replicas=8656 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[38570, 31365, 28429, 32621, 32779, 32764, 32732, 32884] avg=32768.0000 max=38570 max_rank=0 min=28429 min_rank=2 imbalance_ratio=1.177063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32172, 33096, 32621, 32779, 32764, 32732, 32884] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=5474 max_load_reduction_pct=0.141924 moved_to_replicas=8656 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[32778, 32779, 33096, 31789, 32416, 33094, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31789 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32414 max_load_reduction_pct=0.494795 moved_to_replicas=122025 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[38570, 31365, 28429, 32621, 32779, 32764, 32732, 32884] avg=32768.0000 max=38570 max_rank=0 min=28429 min_rank=2 imbalance_ratio=1.177063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[32778, 32779, 55560, 23492, 2, 37502, 14521, 65510] avg=32768.0000 max=65510 max_rank=7 min=2 min_rank=4 imbalance_ratio=1.999207 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32172, 33096, 32621, 32779, 32764, 32732, 32884] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=5474 max_load_reduction_pct=0.141924 moved_to_replicas=8656 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[32763, 65488, 32788, 15, 32782, 78, 32761, 65469] avg=32768.0000 max=65488 max_rank=1 min=15 min_rank=3 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[32778, 32779, 55560, 23492, 2, 37502, 14521, 65510] avg=32768.0000 max=65510 max_rank=7 min=2 min_rank=4 imbalance_ratio=1.999207 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[32778, 32779, 55560, 23492, 2, 37502, 14521, 65510] avg=32768.0000 max=65510 max_rank=7 min=2 min_rank=4 imbalance_ratio=1.999207 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[32778, 32779, 55560, 23492, 2, 37502, 14521, 65510] avg=32768.0000 max=65510 max_rank=7 min=2 min_rank=4 imbalance_ratio=1.999207 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32172, 33096, 32621, 32779, 32764, 32732, 32884] avg=32768.0000 max=33096 max_rank=0 min=32172 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=5474 max_load_reduction_pct=0.141924 moved_to_replicas=8656 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[32778, 32779, 33096, 31789, 32416, 33094, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31789 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32414 max_load_reduction_pct=0.494795 moved_to_replicas=122025 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[32763, 33096, 32788, 32407, 32782, 32451, 32761, 33096] avg=32768.0000 max=33096 max_rank=1 min=32407 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=129559 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[32763, 65488, 32788, 15, 32782, 78, 32761, 65469] avg=32768.0000 max=65488 max_rank=1 min=15 min_rank=3 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[32778, 32779, 33096, 31789, 32416, 33094, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31789 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32414 max_load_reduction_pct=0.494795 moved_to_replicas=122025 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[32778, 32779, 33096, 31789, 32416, 33094, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31789 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32414 max_load_reduction_pct=0.494795 moved_to_replicas=122025 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[32778, 32779, 55560, 23492, 2, 37502, 14521, 65510] avg=32768.0000 max=65510 max_rank=7 min=2 min_rank=4 imbalance_ratio=1.999207 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[32778, 32779, 33096, 31789, 32416, 33094, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31789 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32414 max_load_reduction_pct=0.494795 moved_to_replicas=122025 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[32763, 33096, 32788, 32407, 32782, 32451, 32761, 33096] avg=32768.0000 max=33096 max_rank=1 min=32407 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=129559 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[32763, 65488, 32788, 15, 32782, 78, 32761, 65469] avg=32768.0000 max=65488 max_rank=1 min=15 min_rank=3 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[32778, 32779, 55560, 23492, 2, 37502, 14521, 65510] avg=32768.0000 max=65510 max_rank=7 min=2 min_rank=4 imbalance_ratio=1.999207 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[65068, 32754, 304, 12, 33020, 65460, 65520, 6] avg=32768.0000 max=65520 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[32778, 32779, 33096, 31789, 32416, 33094, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31789 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32414 max_load_reduction_pct=0.494795 moved_to_replicas=122025 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[32763, 65488, 32788, 15, 32782, 78, 32761, 65469] avg=32768.0000 max=65488 max_rank=1 min=15 min_rank=3 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[32763, 65488, 32788, 15, 32782, 78, 32761, 65469] avg=32768.0000 max=65488 max_rank=1 min=15 min_rank=3 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[32763, 65488, 32788, 15, 32782, 78, 32761, 65469] avg=32768.0000 max=65488 max_rank=1 min=15 min_rank=3 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[32763, 33096, 32788, 32407, 32782, 32451, 32761, 33096] avg=32768.0000 max=33096 max_rank=1 min=32407 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=129559 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32754, 32276, 32376, 33020, 33096, 33096, 32430] avg=32768.0000 max=33096 max_rank=0 min=32276 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32424 max_load_reduction_pct=0.494872 moved_to_replicas=193564 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[32778, 32779, 33096, 31789, 32416, 33094, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31789 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32414 max_load_reduction_pct=0.494795 moved_to_replicas=122025 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[32763, 33096, 32788, 32407, 32782, 32451, 32761, 33096] avg=32768.0000 max=33096 max_rank=1 min=32407 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=129559 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[65068, 32754, 304, 12, 33020, 65460, 65520, 6] avg=32768.0000 max=65520 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[32763, 65488, 32788, 15, 32782, 78, 32761, 65469] avg=32768.0000 max=65488 max_rank=1 min=15 min_rank=3 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[32763, 33096, 32788, 32407, 32782, 32451, 32761, 33096] avg=32768.0000 max=33096 max_rank=1 min=32407 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=129559 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[32763, 33096, 32788, 32407, 32782, 32451, 32761, 33096] avg=32768.0000 max=33096 max_rank=1 min=32407 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=129559 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32754, 32276, 32376, 33020, 33096, 33096, 32430] avg=32768.0000 max=33096 max_rank=0 min=32276 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32424 max_load_reduction_pct=0.494872 moved_to_replicas=193564 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[65068, 32754, 304, 12, 33020, 65460, 65520, 6] avg=32768.0000 max=65520 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[66942, 12, 32770, 32751, 22, 62791, 32458, 34398] avg=32768.0000 max=66942 max_rank=0 min=12 min_rank=1 imbalance_ratio=2.042908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[32763, 33096, 32788, 32407, 32782, 32451, 32761, 33096] avg=32768.0000 max=33096 max_rank=1 min=32407 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=129559 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[32763, 65488, 32788, 15, 32782, 78, 32761, 65469] avg=32768.0000 max=65488 max_rank=1 min=15 min_rank=3 imbalance_ratio=1.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[65068, 32754, 304, 12, 33020, 65460, 65520, 6] avg=32768.0000 max=65520 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[65068, 32754, 304, 12, 33020, 65460, 65520, 6] avg=32768.0000 max=65520 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[65068, 32754, 304, 12, 33020, 65460, 65520, 6] avg=32768.0000 max=65520 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32754, 32276, 32376, 33020, 33096, 33096, 32430] avg=32768.0000 max=33096 max_rank=0 min=32276 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32424 max_load_reduction_pct=0.494872 moved_to_replicas=193564 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[33096, 32774, 32770, 32751, 32103, 33096, 32458, 33096] avg=32768.0000 max=33096 max_rank=0 min=32103 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33846 max_load_reduction_pct=0.505602 moved_to_replicas=129864 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32754, 32276, 32376, 33020, 33096, 33096, 32430] avg=32768.0000 max=33096 max_rank=0 min=32276 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32424 max_load_reduction_pct=0.494872 moved_to_replicas=193564 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[32763, 33096, 32788, 32407, 32782, 32451, 32761, 33096] avg=32768.0000 max=33096 max_rank=1 min=32407 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32392 max_load_reduction_pct=0.494625 moved_to_replicas=129559 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[66942, 12, 32770, 32751, 22, 62791, 32458, 34398] avg=32768.0000 max=66942 max_rank=0 min=12 min_rank=1 imbalance_ratio=2.042908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[65068, 32754, 304, 12, 33020, 65460, 65520, 6] avg=32768.0000 max=65520 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32754, 32276, 32376, 33020, 33096, 33096, 32430] avg=32768.0000 max=33096 max_rank=0 min=32276 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32424 max_load_reduction_pct=0.494872 moved_to_replicas=193564 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32754, 32276, 32376, 33020, 33096, 33096, 32430] avg=32768.0000 max=33096 max_rank=0 min=32276 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32424 max_load_reduction_pct=0.494872 moved_to_replicas=193564 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[33096, 32774, 32770, 32751, 32103, 33096, 32458, 33096] avg=32768.0000 max=33096 max_rank=0 min=32103 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33846 max_load_reduction_pct=0.505602 moved_to_replicas=129864 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[66942, 12, 32770, 32751, 22, 62791, 32458, 34398] avg=32768.0000 max=66942 max_rank=0 min=12 min_rank=1 imbalance_ratio=2.042908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[32777, 65498, 65520, 82, 66049, 32194, 19, 5] avg=32768.0000 max=66049 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.015656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[66942, 12, 32770, 32751, 22, 62791, 32458, 34398] avg=32768.0000 max=66942 max_rank=0 min=12 min_rank=1 imbalance_ratio=2.042908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32754, 32276, 32376, 33020, 33096, 33096, 32430] avg=32768.0000 max=33096 max_rank=0 min=32276 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32424 max_load_reduction_pct=0.494872 moved_to_replicas=193564 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[65068, 32754, 304, 12, 33020, 65460, 65520, 6] avg=32768.0000 max=65520 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.999512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[66942, 12, 32770, 32751, 22, 62791, 32458, 34398] avg=32768.0000 max=66942 max_rank=0 min=12 min_rank=1 imbalance_ratio=2.042908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[66942, 12, 32770, 32751, 22, 62791, 32458, 34398] avg=32768.0000 max=66942 max_rank=0 min=12 min_rank=1 imbalance_ratio=2.042908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[33096, 32774, 32770, 32751, 32103, 33096, 32458, 33096] avg=32768.0000 max=33096 max_rank=0 min=32103 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33846 max_load_reduction_pct=0.505602 moved_to_replicas=129864 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[32777, 33096, 33096, 32506, 33096, 32194, 32624, 32755] avg=32768.0000 max=33096 max_rank=1 min=32194 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32953 max_load_reduction_pct=0.498917 moved_to_replicas=195563 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[33096, 32774, 32770, 32751, 32103, 33096, 32458, 33096] avg=32768.0000 max=33096 max_rank=0 min=32103 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33846 max_load_reduction_pct=0.505602 moved_to_replicas=129864 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[32777, 65498, 65520, 82, 66049, 32194, 19, 5] avg=32768.0000 max=66049 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.015656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32754, 32276, 32376, 33020, 33096, 33096, 32430] avg=32768.0000 max=33096 max_rank=0 min=32276 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32424 max_load_reduction_pct=0.494872 moved_to_replicas=193564 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[33096, 32774, 32770, 32751, 32103, 33096, 32458, 33096] avg=32768.0000 max=33096 max_rank=0 min=32103 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33846 max_load_reduction_pct=0.505602 moved_to_replicas=129864 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[66942, 12, 32770, 32751, 22, 62791, 32458, 34398] avg=32768.0000 max=66942 max_rank=0 min=12 min_rank=1 imbalance_ratio=2.042908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[33096, 32774, 32770, 32751, 32103, 33096, 32458, 33096] avg=32768.0000 max=33096 max_rank=0 min=32103 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33846 max_load_reduction_pct=0.505602 moved_to_replicas=129864 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[32777, 33096, 33096, 32506, 33096, 32194, 32624, 32755] avg=32768.0000 max=33096 max_rank=1 min=32194 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32953 max_load_reduction_pct=0.498917 moved_to_replicas=195563 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[32777, 65498, 65520, 82, 66049, 32194, 19, 5] avg=32768.0000 max=66049 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.015656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[18, 32816, 32769, 32767, 32769, 65543, 65445, 17] avg=32768.0000 max=65543 max_rank=5 min=17 min_rank=7 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[32777, 65498, 65520, 82, 66049, 32194, 19, 5] avg=32768.0000 max=66049 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.015656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[66942, 12, 32770, 32751, 22, 62791, 32458, 34398] avg=32768.0000 max=66942 max_rank=0 min=12 min_rank=1 imbalance_ratio=2.042908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[33096, 32774, 32770, 32751, 32103, 33096, 32458, 33096] avg=32768.0000 max=33096 max_rank=0 min=32103 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33846 max_load_reduction_pct=0.505602 moved_to_replicas=129864 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[32777, 33096, 33096, 32506, 33096, 32194, 32624, 32755] avg=32768.0000 max=33096 max_rank=1 min=32194 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32953 max_load_reduction_pct=0.498917 moved_to_replicas=195563 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[32777, 65498, 65520, 82, 66049, 32194, 19, 5] avg=32768.0000 max=66049 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.015656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[32777, 65498, 65520, 82, 66049, 32194, 19, 5] avg=32768.0000 max=66049 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.015656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[32367, 32816, 32769, 32767, 32769, 33096, 33096, 32464] avg=32768.0000 max=33096 max_rank=5 min=32367 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=129561 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[32777, 33096, 33096, 32506, 33096, 32194, 32624, 32755] avg=32768.0000 max=33096 max_rank=1 min=32194 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32953 max_load_reduction_pct=0.498917 moved_to_replicas=195563 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[18, 32816, 32769, 32767, 32769, 65543, 65445, 17] avg=32768.0000 max=65543 max_rank=5 min=17 min_rank=7 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[33096, 32774, 32770, 32751, 32103, 33096, 32458, 33096] avg=32768.0000 max=33096 max_rank=0 min=32103 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33846 max_load_reduction_pct=0.505602 moved_to_replicas=129864 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[32777, 65498, 65520, 82, 66049, 32194, 19, 5] avg=32768.0000 max=66049 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.015656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[32777, 33096, 33096, 32506, 33096, 32194, 32624, 32755] avg=32768.0000 max=33096 max_rank=1 min=32194 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32953 max_load_reduction_pct=0.498917 moved_to_replicas=195563 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[32777, 33096, 33096, 32506, 33096, 32194, 32624, 32755] avg=32768.0000 max=33096 max_rank=1 min=32194 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32953 max_load_reduction_pct=0.498917 moved_to_replicas=195563 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[32367, 32816, 32769, 32767, 32769, 33096, 33096, 32464] avg=32768.0000 max=33096 max_rank=5 min=32367 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=129561 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[18, 32816, 32769, 32767, 32769, 65543, 65445, 17] avg=32768.0000 max=65543 max_rank=5 min=17 min_rank=7 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[65475, 20, 32796, 15, 32774, 12, 65204, 65848] avg=32768.0000 max=65848 max_rank=7 min=12 min_rank=5 imbalance_ratio=2.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[18, 32816, 32769, 32767, 32769, 65543, 65445, 17] avg=32768.0000 max=65543 max_rank=5 min=17 min_rank=7 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[32777, 33096, 33096, 32506, 33096, 32194, 32624, 32755] avg=32768.0000 max=33096 max_rank=1 min=32194 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32953 max_load_reduction_pct=0.498917 moved_to_replicas=195563 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[32777, 65498, 65520, 82, 66049, 32194, 19, 5] avg=32768.0000 max=66049 max_rank=4 min=5 min_rank=7 imbalance_ratio=2.015656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[32367, 32816, 32769, 32767, 32769, 33096, 33096, 32464] avg=32768.0000 max=33096 max_rank=5 min=32367 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=129561 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[18, 32816, 32769, 32767, 32769, 65543, 65445, 17] avg=32768.0000 max=65543 max_rank=5 min=17 min_rank=7 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[18, 32816, 32769, 32767, 32769, 65543, 65445, 17] avg=32768.0000 max=65543 max_rank=5 min=17 min_rank=7 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[33096, 32128, 32796, 32394, 32774, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32128 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32752 max_load_reduction_pct=0.497388 moved_to_replicas=194478 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[32367, 32816, 32769, 32767, 32769, 33096, 33096, 32464] avg=32768.0000 max=33096 max_rank=5 min=32367 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=129561 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[65475, 20, 32796, 15, 32774, 12, 65204, 65848] avg=32768.0000 max=65848 max_rank=7 min=12 min_rank=5 imbalance_ratio=2.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[32777, 33096, 33096, 32506, 33096, 32194, 32624, 32755] avg=32768.0000 max=33096 max_rank=1 min=32194 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32953 max_load_reduction_pct=0.498917 moved_to_replicas=195563 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[18, 32816, 32769, 32767, 32769, 65543, 65445, 17] avg=32768.0000 max=65543 max_rank=5 min=17 min_rank=7 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[32367, 32816, 32769, 32767, 32769, 33096, 33096, 32464] avg=32768.0000 max=33096 max_rank=5 min=32367 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=129561 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[32367, 32816, 32769, 32767, 32769, 33096, 33096, 32464] avg=32768.0000 max=33096 max_rank=5 min=32367 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=129561 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[33096, 32128, 32796, 32394, 32774, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32128 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32752 max_load_reduction_pct=0.497388 moved_to_replicas=194478 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[65475, 20, 32796, 15, 32774, 12, 65204, 65848] avg=32768.0000 max=65848 max_rank=7 min=12 min_rank=5 imbalance_ratio=2.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[32841, 98232, 25, 32771, 32714, 15, 32755, 32791] avg=32768.0000 max=98232 max_rank=1 min=15 min_rank=5 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[65475, 20, 32796, 15, 32774, 12, 65204, 65848] avg=32768.0000 max=65848 max_rank=7 min=12 min_rank=5 imbalance_ratio=2.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[32367, 32816, 32769, 32767, 32769, 33096, 33096, 32464] avg=32768.0000 max=33096 max_rank=5 min=32367 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=129561 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[18, 32816, 32769, 32767, 32769, 65543, 65445, 17] avg=32768.0000 max=65543 max_rank=5 min=17 min_rank=7 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[33096, 32128, 32796, 32394, 32774, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32128 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32752 max_load_reduction_pct=0.497388 moved_to_replicas=194478 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[65475, 20, 32796, 15, 32774, 12, 65204, 65848] avg=32768.0000 max=65848 max_rank=7 min=12 min_rank=5 imbalance_ratio=2.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[32841, 33096, 32402, 32771, 32714, 32774, 32755, 32791] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=130226 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[65475, 20, 32796, 15, 32774, 12, 65204, 65848] avg=32768.0000 max=65848 max_rank=7 min=12 min_rank=5 imbalance_ratio=2.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[33096, 32128, 32796, 32394, 32774, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32128 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32752 max_load_reduction_pct=0.497388 moved_to_replicas=194478 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[32841, 98232, 25, 32771, 32714, 15, 32755, 32791] avg=32768.0000 max=98232 max_rank=1 min=15 min_rank=5 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[32367, 32816, 32769, 32767, 32769, 33096, 33096, 32464] avg=32768.0000 max=33096 max_rank=5 min=32367 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=129561 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[33096, 32128, 32796, 32394, 32774, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32128 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32752 max_load_reduction_pct=0.497388 moved_to_replicas=194478 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[65475, 20, 32796, 15, 32774, 12, 65204, 65848] avg=32768.0000 max=65848 max_rank=7 min=12 min_rank=5 imbalance_ratio=2.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[33096, 32128, 32796, 32394, 32774, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32128 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32752 max_load_reduction_pct=0.497388 moved_to_replicas=194478 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[32841, 33096, 32402, 32771, 32714, 32774, 32755, 32791] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=130226 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[32841, 98232, 25, 32771, 32714, 15, 32755, 32791] avg=32768.0000 max=98232 max_rank=1 min=15 min_rank=5 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[10, 31152, 64939, 32894, 1371, 65474, 32777, 33527] avg=32768.0000 max=65474 max_rank=5 min=10 min_rank=0 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[32841, 98232, 25, 32771, 32714, 15, 32755, 32791] avg=32768.0000 max=98232 max_rank=1 min=15 min_rank=5 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[33096, 32128, 32796, 32394, 32774, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32128 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32752 max_load_reduction_pct=0.497388 moved_to_replicas=194478 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[65475, 20, 32796, 15, 32774, 12, 65204, 65848] avg=32768.0000 max=65848 max_rank=7 min=12 min_rank=5 imbalance_ratio=2.009521 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[32841, 33096, 32402, 32771, 32714, 32774, 32755, 32791] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=130226 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[32841, 98232, 25, 32771, 32714, 15, 32755, 32791] avg=32768.0000 max=98232 max_rank=1 min=15 min_rank=5 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[32388, 31701, 33096, 32894, 33096, 33096, 32777, 33096] avg=32768.0000 max=33096 max_rank=2 min=31701 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=129314 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[32841, 33096, 32402, 32771, 32714, 32774, 32755, 32791] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=130226 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[32841, 98232, 25, 32771, 32714, 15, 32755, 32791] avg=32768.0000 max=98232 max_rank=1 min=15 min_rank=5 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[10, 31152, 64939, 32894, 1371, 65474, 32777, 33527] avg=32768.0000 max=65474 max_rank=5 min=10 min_rank=0 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[33096, 32128, 32796, 32394, 32774, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32128 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32752 max_load_reduction_pct=0.497388 moved_to_replicas=194478 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[32841, 33096, 32402, 32771, 32714, 32774, 32755, 32791] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=130226 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[32841, 98232, 25, 32771, 32714, 15, 32755, 32791] avg=32768.0000 max=98232 max_rank=1 min=15 min_rank=5 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[32841, 33096, 32402, 32771, 32714, 32774, 32755, 32791] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=130226 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[32388, 31701, 33096, 32894, 33096, 33096, 32777, 33096] avg=32768.0000 max=33096 max_rank=2 min=31701 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=129314 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[10, 31152, 64939, 32894, 1371, 65474, 32777, 33527] avg=32768.0000 max=65474 max_rank=5 min=10 min_rank=0 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[65337, 32709, 32774, 221, 44, 32775, 32771, 65513] avg=32768.0000 max=65513 max_rank=7 min=44 min_rank=4 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[10, 31152, 64939, 32894, 1371, 65474, 32777, 33527] avg=32768.0000 max=65474 max_rank=5 min=10 min_rank=0 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[32841, 98232, 25, 32771, 32714, 15, 32755, 32791] avg=32768.0000 max=98232 max_rank=1 min=15 min_rank=5 imbalance_ratio=2.997803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[32841, 33096, 32402, 32771, 32714, 32774, 32755, 32791] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=130226 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[32388, 31701, 33096, 32894, 33096, 33096, 32777, 33096] avg=32768.0000 max=33096 max_rank=2 min=31701 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=129314 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[33096, 32709, 32774, 32462, 32461, 32775, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32461 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=129237 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[10, 31152, 64939, 32894, 1371, 65474, 32777, 33527] avg=32768.0000 max=65474 max_rank=5 min=10 min_rank=0 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[32388, 31701, 33096, 32894, 33096, 33096, 32777, 33096] avg=32768.0000 max=33096 max_rank=2 min=31701 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=129314 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[65337, 32709, 32774, 221, 44, 32775, 32771, 65513] avg=32768.0000 max=65513 max_rank=7 min=44 min_rank=4 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[10, 31152, 64939, 32894, 1371, 65474, 32777, 33527] avg=32768.0000 max=65474 max_rank=5 min=10 min_rank=0 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[32841, 33096, 32402, 32771, 32714, 32774, 32755, 32791] avg=32768.0000 max=33096 max_rank=1 min=32402 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65136 max_load_reduction_pct=0.663083 moved_to_replicas=130226 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[32388, 31701, 33096, 32894, 33096, 33096, 32777, 33096] avg=32768.0000 max=33096 max_rank=2 min=31701 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=129314 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[33096, 32709, 32774, 32462, 32461, 32775, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32461 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=129237 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[10, 31152, 64939, 32894, 1371, 65474, 32777, 33527] avg=32768.0000 max=65474 max_rank=5 min=10 min_rank=0 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[32388, 31701, 33096, 32894, 33096, 33096, 32777, 33096] avg=32768.0000 max=33096 max_rank=2 min=31701 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=129314 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[39, 77517, 114, 32754, 65506, 20683, 32765, 32766] avg=32768.0000 max=77517 max_rank=1 min=39 min_rank=0 imbalance_ratio=2.365631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[65337, 32709, 32774, 221, 44, 32775, 32771, 65513] avg=32768.0000 max=65513 max_rank=7 min=44 min_rank=4 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[65337, 32709, 32774, 221, 44, 32775, 32771, 65513] avg=32768.0000 max=65513 max_rank=7 min=44 min_rank=4 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[10, 31152, 64939, 32894, 1371, 65474, 32777, 33527] avg=32768.0000 max=65474 max_rank=5 min=10 min_rank=0 imbalance_ratio=1.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[32388, 31701, 33096, 32894, 33096, 33096, 32777, 33096] avg=32768.0000 max=33096 max_rank=2 min=31701 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=129314 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[32786, 33096, 33096, 32754, 33096, 31785, 32765, 32766] avg=32768.0000 max=33096 max_rank=1 min=31785 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44421 max_load_reduction_pct=0.573048 moved_to_replicas=159146 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[33096, 32709, 32774, 32462, 32461, 32775, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32461 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=129237 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[65337, 32709, 32774, 221, 44, 32775, 32771, 65513] avg=32768.0000 max=65513 max_rank=7 min=44 min_rank=4 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[33096, 32709, 32774, 32462, 32461, 32775, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32461 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=129237 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[39, 77517, 114, 32754, 65506, 20683, 32765, 32766] avg=32768.0000 max=77517 max_rank=1 min=39 min_rank=0 imbalance_ratio=2.365631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[65337, 32709, 32774, 221, 44, 32775, 32771, 65513] avg=32768.0000 max=65513 max_rank=7 min=44 min_rank=4 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[32388, 31701, 33096, 32894, 33096, 33096, 32777, 33096] avg=32768.0000 max=33096 max_rank=2 min=31701 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32378 max_load_reduction_pct=0.494517 moved_to_replicas=129314 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[33096, 32709, 32774, 32462, 32461, 32775, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32461 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=129237 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[65337, 32709, 32774, 221, 44, 32775, 32771, 65513] avg=32768.0000 max=65513 max_rank=7 min=44 min_rank=4 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[32786, 33096, 33096, 32754, 33096, 31785, 32765, 32766] avg=32768.0000 max=33096 max_rank=1 min=31785 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44421 max_load_reduction_pct=0.573048 moved_to_replicas=159146 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[33096, 32709, 32774, 32462, 32461, 32775, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32461 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=129237 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[18, 32780, 65472, 32766, 13, 32793, 32776, 65526] avg=32768.0000 max=65526 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[39, 77517, 114, 32754, 65506, 20683, 32765, 32766] avg=32768.0000 max=77517 max_rank=1 min=39 min_rank=0 imbalance_ratio=2.365631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[39, 77517, 114, 32754, 65506, 20683, 32765, 32766] avg=32768.0000 max=77517 max_rank=1 min=39 min_rank=0 imbalance_ratio=2.365631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[65337, 32709, 32774, 221, 44, 32775, 32771, 65513] avg=32768.0000 max=65513 max_rank=7 min=44 min_rank=4 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[33096, 32709, 32774, 32462, 32461, 32775, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32461 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=129237 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[32394, 32780, 33096, 32766, 32443, 32793, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=129510 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[32786, 33096, 33096, 32754, 33096, 31785, 32765, 32766] avg=32768.0000 max=33096 max_rank=1 min=31785 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44421 max_load_reduction_pct=0.573048 moved_to_replicas=159146 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[39, 77517, 114, 32754, 65506, 20683, 32765, 32766] avg=32768.0000 max=77517 max_rank=1 min=39 min_rank=0 imbalance_ratio=2.365631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[32786, 33096, 33096, 32754, 33096, 31785, 32765, 32766] avg=32768.0000 max=33096 max_rank=1 min=31785 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44421 max_load_reduction_pct=0.573048 moved_to_replicas=159146 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[18, 32780, 65472, 32766, 13, 32793, 32776, 65526] avg=32768.0000 max=65526 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[39, 77517, 114, 32754, 65506, 20683, 32765, 32766] avg=32768.0000 max=77517 max_rank=1 min=39 min_rank=0 imbalance_ratio=2.365631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[33096, 32709, 32774, 32462, 32461, 32775, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32461 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=129237 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[32786, 33096, 33096, 32754, 33096, 31785, 32765, 32766] avg=32768.0000 max=33096 max_rank=1 min=31785 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44421 max_load_reduction_pct=0.573048 moved_to_replicas=159146 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[39, 77517, 114, 32754, 65506, 20683, 32765, 32766] avg=32768.0000 max=77517 max_rank=1 min=39 min_rank=0 imbalance_ratio=2.365631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[32766, 8, 32788, 32758, 65523, 32779, 65472, 50] avg=32768.0000 max=65523 max_rank=4 min=8 min_rank=1 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[32786, 33096, 33096, 32754, 33096, 31785, 32765, 32766] avg=32768.0000 max=33096 max_rank=1 min=31785 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44421 max_load_reduction_pct=0.573048 moved_to_replicas=159146 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[32394, 32780, 33096, 32766, 32443, 32793, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=129510 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[18, 32780, 65472, 32766, 13, 32793, 32776, 65526] avg=32768.0000 max=65526 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[18, 32780, 65472, 32766, 13, 32793, 32776, 65526] avg=32768.0000 max=65526 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[39, 77517, 114, 32754, 65506, 20683, 32765, 32766] avg=32768.0000 max=77517 max_rank=1 min=39 min_rank=0 imbalance_ratio=2.365631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[32766, 32435, 32788, 32758, 33096, 32779, 33096, 32426] avg=32768.0000 max=33096 max_rank=4 min=32426 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=129604 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[32786, 33096, 33096, 32754, 33096, 31785, 32765, 32766] avg=32768.0000 max=33096 max_rank=1 min=31785 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44421 max_load_reduction_pct=0.573048 moved_to_replicas=159146 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[32394, 32780, 33096, 32766, 32443, 32793, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=129510 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[18, 32780, 65472, 32766, 13, 32793, 32776, 65526] avg=32768.0000 max=65526 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[32394, 32780, 33096, 32766, 32443, 32793, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=129510 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[32786, 33096, 33096, 32754, 33096, 31785, 32765, 32766] avg=32768.0000 max=33096 max_rank=1 min=31785 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44421 max_load_reduction_pct=0.573048 moved_to_replicas=159146 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[18, 32780, 65472, 32766, 13, 32793, 32776, 65526] avg=32768.0000 max=65526 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[32766, 8, 32788, 32758, 65523, 32779, 65472, 50] avg=32768.0000 max=65523 max_rank=4 min=8 min_rank=1 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[32394, 32780, 33096, 32766, 32443, 32793, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=129510 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[65510, 32800, 0, 30, 32768, 20, 65503, 65513] avg=32768.0000 max=65513 max_rank=7 min=0 min_rank=2 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[18, 32780, 65472, 32766, 13, 32793, 32776, 65526] avg=32768.0000 max=65526 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[32394, 32780, 33096, 32766, 32443, 32793, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=129510 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[32766, 8, 32788, 32758, 65523, 32779, 65472, 50] avg=32768.0000 max=65523 max_rank=4 min=8 min_rank=1 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[32766, 32435, 32788, 32758, 33096, 32779, 33096, 32426] avg=32768.0000 max=33096 max_rank=4 min=32426 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=129604 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[32766, 8, 32788, 32758, 65523, 32779, 65472, 50] avg=32768.0000 max=65523 max_rank=4 min=8 min_rank=1 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[18, 32780, 65472, 32766, 13, 32793, 32776, 65526] avg=32768.0000 max=65526 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[33096, 32800, 32417, 32437, 32768, 32434, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32417 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=194455 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[32394, 32780, 33096, 32766, 32443, 32793, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=129510 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[32766, 8, 32788, 32758, 65523, 32779, 65472, 50] avg=32768.0000 max=65523 max_rank=4 min=8 min_rank=1 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[32766, 32435, 32788, 32758, 33096, 32779, 33096, 32426] avg=32768.0000 max=33096 max_rank=4 min=32426 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=129604 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[32766, 32435, 32788, 32758, 33096, 32779, 33096, 32426] avg=32768.0000 max=33096 max_rank=4 min=32426 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=129604 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[32394, 32780, 33096, 32766, 32443, 32793, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=32394 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=129510 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[32766, 8, 32788, 32758, 65523, 32779, 65472, 50] avg=32768.0000 max=65523 max_rank=4 min=8 min_rank=1 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[65510, 32800, 0, 30, 32768, 20, 65503, 65513] avg=32768.0000 max=65513 max_rank=7 min=0 min_rank=2 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[32766, 32435, 32788, 32758, 33096, 32779, 33096, 32426] avg=32768.0000 max=33096 max_rank=4 min=32426 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=129604 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[32766, 8, 32788, 32758, 65523, 32779, 65472, 50] avg=32768.0000 max=65523 max_rank=4 min=8 min_rank=1 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[32766, 32435, 32788, 32758, 33096, 32779, 33096, 32426] avg=32768.0000 max=33096 max_rank=4 min=32426 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=129604 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[33096, 32800, 32417, 32437, 32768, 32434, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32417 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=194455 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[65510, 32800, 0, 30, 32768, 20, 65503, 65513] avg=32768.0000 max=65513 max_rank=7 min=0 min_rank=2 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[65510, 32800, 0, 30, 32768, 20, 65503, 65513] avg=32768.0000 max=65513 max_rank=7 min=0 min_rank=2 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[32766, 8, 32788, 32758, 65523, 32779, 65472, 50] avg=32768.0000 max=65523 max_rank=4 min=8 min_rank=1 imbalance_ratio=1.999603 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[32766, 32435, 32788, 32758, 33096, 32779, 33096, 32426] avg=32768.0000 max=33096 max_rank=4 min=32426 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=129604 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[65510, 32800, 0, 30, 32768, 20, 65503, 65513] avg=32768.0000 max=65513 max_rank=7 min=0 min_rank=2 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[33096, 32800, 32417, 32437, 32768, 32434, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32417 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=194455 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[33096, 32800, 32417, 32437, 32768, 32434, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32417 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=194455 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[32766, 32435, 32788, 32758, 33096, 32779, 33096, 32426] avg=32768.0000 max=33096 max_rank=4 min=32426 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32427 max_load_reduction_pct=0.494895 moved_to_replicas=129604 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[65510, 32800, 0, 30, 32768, 20, 65503, 65513] avg=32768.0000 max=65513 max_rank=7 min=0 min_rank=2 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[33096, 32800, 32417, 32437, 32768, 32434, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32417 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=194455 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[33096, 32800, 32417, 32437, 32768, 32434, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32417 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=194455 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[65510, 32800, 0, 30, 32768, 20, 65503, 65513] avg=32768.0000 max=65513 max_rank=7 min=0 min_rank=2 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[65510, 32800, 0, 30, 32768, 20, 65503, 65513] avg=32768.0000 max=65513 max_rank=7 min=0 min_rank=2 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[33096, 32800, 32417, 32437, 32768, 32434, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32417 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=194455 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[33096, 32800, 32417, 32437, 32768, 32434, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32417 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=194455 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=32 total_expected_recv=1048776 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=55808 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=32 total_expected_recv=1051052 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=53120 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=32 total_expected_recv=1049280 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=54912 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=32 total_expected_recv=1043512 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=59776 peak_demand=33095 required_capacity=1.009979 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=32 total_expected_recv=1044314 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=59904 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=32 total_expected_recv=1047399 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56448 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=32 total_expected_recv=1051709 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=51712 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=32 total_expected_recv=1052566 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=53248 peak_demand=33096 required_capacity=1.010010 + [2026-08-30 05:11:43] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 527.0 | throughput per GPU (TFLOP/s/GPU): 632.6 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.047652E+01 | loss scale: 1.0 | grad norm: 38.827 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32282, 33096, 33097, 33096, 31286, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=34816 expected_recv=65377 expected_overflow=0 exact_recv_per_rank=[33096, 32282, 33096, 33097, 33096, 31286, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=34816 expected_recv=64382 expected_overflow=0 exact_recv_per_rank=[33096, 32282, 33096, 33097, 33096, 31286, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 32282, 33096, 33097, 33096, 31286, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 32282, 33096, 33097, 33096, 31286, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=34816 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[33096, 32282, 33096, 33097, 33096, 31286, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=34816 expected_recv=65377 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32282, 33096, 33097, 33096, 31286, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=34816 expected_recv=64382 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 32282, 33096, 33097, 33096, 31286, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=34816 expected_recv=63568 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=64256 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33097, 32782, 33096, 32790, 31884, 32303, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=34816 expected_recv=65600 expected_overflow=0 exact_recv_per_rank=[33097, 32782, 33096, 32790, 31884, 32303, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=34816 expected_recv=63459 expected_overflow=0 exact_recv_per_rank=[33097, 32782, 33096, 32790, 31884, 32303, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=34816 expected_recv=65561 expected_overflow=0 exact_recv_per_rank=[33097, 32782, 33096, 32790, 31884, 32303, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 32782, 33096, 32790, 31884, 32303, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=34816 expected_recv=65600 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=34816 expected_recv=63459 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=34816 expected_recv=64980 expected_overflow=0 exact_recv_per_rank=[33097, 32782, 33096, 32790, 31884, 32303, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 32782, 33096, 32790, 31884, 32303, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=34816 expected_recv=65561 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=64384 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=34816 expected_recv=64980 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=34816 expected_recv=66111 expected_overflow=0 exact_recv_per_rank=[33097, 32782, 33096, 32790, 31884, 32303, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31799, 32769, 33098, 33096, 32447, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=34816 expected_recv=66111 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31799, 32769, 33098, 33096, 32447, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31799, 32769, 33098, 33096, 32447, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=34816 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31799, 32769, 33098, 33096, 32447, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=34816 expected_recv=65157 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31799, 32769, 33098, 33096, 32447, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=34816 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31799, 32769, 33098, 33096, 32447, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=34816 expected_recv=65536 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=34816 expected_recv=65157 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=34816 expected_recv=65840 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31799, 32769, 33098, 33096, 32447, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=34816 expected_recv=65545 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=34816 expected_recv=65840 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=34816 expected_recv=63635 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 31799, 32769, 33098, 33096, 32447, 32744] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32769, 32444, 32834, 33096, 32040, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=34816 expected_recv=63635 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=34816 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32769, 32444, 32834, 33096, 32040, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32769, 32444, 32834, 33096, 32040, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=64512 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=34816 expected_recv=65542 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=34816 expected_recv=64881 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32769, 32444, 32834, 33096, 32040, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=34816 expected_recv=64079 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32769, 32444, 32834, 33096, 32040, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=34816 expected_recv=65666 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32769, 32444, 32834, 33096, 32040, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=34816 expected_recv=64881 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=34816 expected_recv=65666 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=34816 expected_recv=64079 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32769, 32444, 32834, 33096, 32040, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=34816 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32769, 32444, 32834, 33096, 32040, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=34816 expected_recv=64267 expected_overflow=0 exact_recv_per_rank=[32134, 33096, 33096, 32424, 33096, 32765, 33096, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32134, 33096, 33096, 32424, 33096, 32765, 33096, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=34816 expected_recv=65544 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=34816 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[32134, 33096, 33096, 32424, 33096, 32765, 33096, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=34816 expected_recv=64267 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=34816 expected_recv=65542 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=34816 expected_recv=64839 expected_overflow=0 exact_recv_per_rank=[32134, 33096, 33096, 32424, 33096, 32765, 33096, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32134, 33096, 33096, 32424, 33096, 32765, 33096, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32134, 33096, 33096, 32424, 33096, 32765, 33096, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=34816 expected_recv=64839 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=34816 expected_recv=64872 expected_overflow=0 exact_recv_per_rank=[32134, 33096, 33096, 32424, 33096, 32765, 33096, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32134, 33096, 33096, 32424, 33096, 32765, 33096, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=34816 expected_recv=64872 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32320, 32816, 32833, 32447, 33096, 32769, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=34816 expected_recv=64762 expected_overflow=0 exact_recv_per_rank=[33096, 32320, 32816, 32833, 32447, 33096, 32769, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32320, 32816, 32833, 32447, 33096, 32769, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=34816 expected_recv=65765 expected_overflow=0 exact_recv_per_rank=[33096, 32320, 32816, 32833, 32447, 33096, 32769, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=34816 expected_recv=64762 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=34816 expected_recv=64677 expected_overflow=0 exact_recv_per_rank=[33096, 32320, 32816, 32833, 32447, 33096, 32769, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=34816 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[33096, 32320, 32816, 32833, 32447, 33096, 32769, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=34816 expected_recv=65765 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=34816 expected_recv=64677 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=34816 expected_recv=65536 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=34816 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[33096, 32320, 32816, 32833, 32447, 33096, 32769, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=34816 expected_recv=65541 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=34816 expected_recv=65623 expected_overflow=0 exact_recv_per_rank=[33096, 32320, 32816, 32833, 32447, 33096, 32769, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=34816 expected_recv=64859 expected_overflow=0 exact_recv_per_rank=[32432, 32453, 33096, 32760, 33096, 32772, 32763, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=34816 expected_recv=65623 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=34816 expected_recv=64936 expected_overflow=0 exact_recv_per_rank=[32432, 32453, 33096, 32760, 33096, 32772, 32763, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=34816 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[32432, 32453, 33096, 32760, 33096, 32772, 32763, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=34816 expected_recv=64859 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=34816 expected_recv=64936 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=34816 expected_recv=65532 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32432, 32453, 33096, 32760, 33096, 32772, 32763, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=34816 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[32432, 32453, 33096, 32760, 33096, 32772, 32763, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=34816 expected_recv=65503 expected_overflow=0 exact_recv_per_rank=[32432, 32453, 33096, 32760, 33096, 32772, 32763, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=34816 expected_recv=65532 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=34816 expected_recv=65503 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=34816 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[32432, 32453, 33096, 32760, 33096, 32772, 32763, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32432, 32453, 33096, 32760, 33096, 32772, 32763, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=34816 expected_recv=65542 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=34816 expected_recv=64880 expected_overflow=0 exact_recv_per_rank=[32432, 33096, 32684, 33096, 32766, 32870, 32763, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=34816 expected_recv=65937 expected_overflow=0 exact_recv_per_rank=[32432, 33096, 32684, 33096, 32766, 32870, 32763, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32432, 33096, 32684, 33096, 32766, 32870, 32763, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=34816 expected_recv=64880 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=34816 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[32432, 33096, 32684, 33096, 32766, 32870, 32763, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=34816 expected_recv=65937 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32432, 33096, 32684, 33096, 32766, 32870, 32763, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=34816 expected_recv=65536 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=34816 expected_recv=65509 expected_overflow=0 exact_recv_per_rank=[32432, 33096, 32684, 33096, 32766, 32870, 32763, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=34816 expected_recv=65509 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=34816 expected_recv=64821 expected_overflow=0 exact_recv_per_rank=[32432, 33096, 32684, 33096, 32766, 32870, 32763, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=34816 expected_recv=65221 expected_overflow=0 exact_recv_per_rank=[32432, 33096, 32684, 33096, 32766, 32870, 32763, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=34816 expected_recv=64821 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=34816 expected_recv=65221 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31860, 33096, 32777, 33096, 32453, 32670, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=34816 expected_recv=64943 expected_overflow=0 exact_recv_per_rank=[33096, 31860, 33096, 32777, 33096, 32453, 32670, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=34816 expected_recv=63727 expected_overflow=0 exact_recv_per_rank=[33096, 31860, 33096, 32777, 33096, 32453, 32670, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31860, 33096, 32777, 33096, 32453, 32670, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=34816 expected_recv=64943 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=34816 expected_recv=65560 expected_overflow=0 exact_recv_per_rank=[33096, 31860, 33096, 32777, 33096, 32453, 32670, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=34816 expected_recv=63727 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=34816 expected_recv=65290 expected_overflow=0 exact_recv_per_rank=[33096, 31860, 33096, 32777, 33096, 32453, 32670, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=34816 expected_recv=65560 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=64384 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=34816 expected_recv=65290 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31860, 33096, 32777, 33096, 32453, 32670, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31860, 33096, 32777, 33096, 32453, 32670, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31731, 32799, 32794, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=34816 expected_recv=65621 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31731, 32799, 32794, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31731, 32799, 32794, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=34816 expected_recv=64164 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31731, 32799, 32794, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31731, 32799, 32794, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=34816 expected_recv=65621 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=34816 expected_recv=64164 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=34816 expected_recv=65566 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31731, 32799, 32794, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31731, 32799, 32794, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=34816 expected_recv=65566 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=34816 expected_recv=64169 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31731, 32799, 32794, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=34816 expected_recv=64169 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65024 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=34816 expected_recv=65553 expected_overflow=0 exact_recv_per_rank=[32775, 32091, 32773, 32767, 33096, 33096, 32777, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32775, 32091, 32773, 32767, 33096, 33096, 32777, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=34816 expected_recv=64165 expected_overflow=0 exact_recv_per_rank=[32775, 32091, 32773, 32767, 33096, 33096, 32777, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32775, 32091, 32773, 32767, 33096, 33096, 32777, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=34816 expected_recv=65553 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=34816 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[32775, 32091, 32773, 32767, 33096, 33096, 32777, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=34816 expected_recv=64165 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=34816 expected_recv=65536 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=64768 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=34816 expected_recv=65566 expected_overflow=0 exact_recv_per_rank=[32775, 32091, 32773, 32767, 33096, 33096, 32777, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=34816 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[32775, 32091, 32773, 32767, 33096, 33096, 32777, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=34816 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[32775, 32091, 32773, 32767, 33096, 33096, 32777, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=34816 expected_recv=65566 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=34816 expected_recv=65538 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=34816 expected_recv=65546 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32771, 32716, 31829, 32773, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=34816 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32771, 32716, 31829, 32773, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32771, 32716, 31829, 32773, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=34816 expected_recv=63665 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32771, 32716, 31829, 32773, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=34816 expected_recv=65546 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=34816 expected_recv=65431 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32771, 32716, 31829, 32773, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=34816 expected_recv=63665 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=34816 expected_recv=65431 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=34816 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32771, 32716, 31829, 32773, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=34816 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32771, 32716, 31829, 32773, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=64768 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32771, 32716, 31829, 32773, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=34816 expected_recv=65533 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=34816 expected_recv=65537 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=34816 expected_recv=64890 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32769, 32774, 33096, 32778, 32420, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=34816 expected_recv=65577 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32769, 32774, 33096, 32778, 32420, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=34816 expected_recv=64890 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32769, 32774, 33096, 32778, 32420, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=34816 expected_recv=65577 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32769, 32774, 33096, 32778, 32420, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=34816 expected_recv=65553 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32769, 32774, 33096, 32778, 32420, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=34816 expected_recv=65553 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=34816 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32769, 32774, 33096, 32778, 32420, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=34816 expected_recv=64832 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32769, 32774, 33096, 32778, 32420, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=34816 expected_recv=65511 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32769, 32774, 33096, 32778, 32420, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=34816 expected_recv=65541 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=34816 expected_recv=64832 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=34816 expected_recv=65511 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=34816 expected_recv=64893 expected_overflow=0 exact_recv_per_rank=[32439, 32776, 33096, 32764, 32453, 32775, 32745, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=34816 expected_recv=65568 expected_overflow=0 exact_recv_per_rank=[32439, 32776, 33096, 32764, 32453, 32775, 32745, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=34816 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[32439, 32776, 33096, 32764, 32453, 32775, 32745, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=34816 expected_recv=64900 expected_overflow=0 exact_recv_per_rank=[32439, 32776, 33096, 32764, 32453, 32775, 32745, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=34816 expected_recv=64893 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=34816 expected_recv=65568 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=34816 expected_recv=65528 expected_overflow=0 exact_recv_per_rank=[32439, 32776, 33096, 32764, 32453, 32775, 32745, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=34816 expected_recv=65535 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=34816 expected_recv=64900 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32439, 32776, 33096, 32764, 32453, 32775, 32745, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=34816 expected_recv=65528 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=34816 expected_recv=65480 expected_overflow=0 exact_recv_per_rank=[32439, 32776, 33096, 32764, 32453, 32775, 32745, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32439, 32776, 33096, 32764, 32453, 32775, 32745, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=34816 expected_recv=65480 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=34816 expected_recv=65549 expected_overflow=0 exact_recv_per_rank=[32774, 32754, 32592, 32776, 32424, 33096, 33096, 32632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32774, 32754, 32592, 32776, 32424, 33096, 33096, 32632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=34816 expected_recv=65503 expected_overflow=0 exact_recv_per_rank=[32774, 32754, 32592, 32776, 32424, 33096, 33096, 32632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=34816 expected_recv=64854 expected_overflow=0 exact_recv_per_rank=[32774, 32754, 32592, 32776, 32424, 33096, 33096, 32632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=34816 expected_recv=65549 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=34816 expected_recv=65503 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=34816 expected_recv=64854 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=34816 expected_recv=65548 expected_overflow=0 exact_recv_per_rank=[32774, 32754, 32592, 32776, 32424, 33096, 33096, 32632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=34816 expected_recv=65270 expected_overflow=0 exact_recv_per_rank=[32774, 32754, 32592, 32776, 32424, 33096, 33096, 32632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=34816 expected_recv=65548 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32774, 32754, 32592, 32776, 32424, 33096, 33096, 32632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=34816 expected_recv=65180 expected_overflow=0 exact_recv_per_rank=[32774, 32754, 32592, 32776, 32424, 33096, 33096, 32632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=34816 expected_recv=65270 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=34816 expected_recv=65180 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32541, 32728, 32440, 32757, 32390, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=34816 expected_recv=64776 expected_overflow=0 exact_recv_per_rank=[33096, 32541, 32728, 32440, 32757, 32390, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=34816 expected_recv=65510 expected_overflow=0 exact_recv_per_rank=[33096, 32541, 32728, 32440, 32757, 32390, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=34816 expected_recv=65140 expected_overflow=0 exact_recv_per_rank=[33096, 32541, 32728, 32440, 32757, 32390, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=34816 expected_recv=64776 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=34816 expected_recv=65510 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=34816 expected_recv=65140 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=34816 expected_recv=64879 expected_overflow=0 exact_recv_per_rank=[33096, 32541, 32728, 32440, 32757, 32390, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=34816 expected_recv=65407 expected_overflow=0 exact_recv_per_rank=[33096, 32541, 32728, 32440, 32757, 32390, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=34816 expected_recv=64879 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32541, 32728, 32440, 32757, 32390, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=34816 expected_recv=65407 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32541, 32728, 32440, 32757, 32390, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[7062, 17750, 39231, 56490, 64973, 18480, 38059, 20099] avg=32768.0000 max=64973 max_rank=4 min=7062 min_rank=0 imbalance_ratio=1.982819 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[7062, 17750, 39231, 56490, 64973, 18480, 38059, 20099] avg=32768.0000 max=64973 max_rank=4 min=7062 min_rank=0 imbalance_ratio=1.982819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[7062, 17750, 39231, 56490, 64973, 18480, 38059, 20099] avg=32768.0000 max=64973 max_rank=4 min=7062 min_rank=0 imbalance_ratio=1.982819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[7062, 17750, 39231, 56490, 64973, 18480, 38059, 20099] avg=32768.0000 max=64973 max_rank=4 min=7062 min_rank=0 imbalance_ratio=1.982819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[33096, 32282, 33096, 33097, 33096, 31286, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31286 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31876 max_load_reduction_pct=0.490604 moved_to_replicas=131976 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[7062, 17750, 39231, 56490, 64973, 18480, 38059, 20099] avg=32768.0000 max=64973 max_rank=4 min=7062 min_rank=0 imbalance_ratio=1.982819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[33096, 32282, 33096, 33097, 33096, 31286, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31286 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31876 max_load_reduction_pct=0.490604 moved_to_replicas=131976 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[7062, 17750, 39231, 56490, 64973, 18480, 38059, 20099] avg=32768.0000 max=64973 max_rank=4 min=7062 min_rank=0 imbalance_ratio=1.982819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[33096, 32282, 33096, 33097, 33096, 31286, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31286 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31876 max_load_reduction_pct=0.490604 moved_to_replicas=131976 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[7062, 17750, 39231, 56490, 64973, 18480, 38059, 20099] avg=32768.0000 max=64973 max_rank=4 min=7062 min_rank=0 imbalance_ratio=1.982819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[33096, 32282, 33096, 33097, 33096, 31286, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31286 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31876 max_load_reduction_pct=0.490604 moved_to_replicas=131976 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[7062, 17750, 39231, 56490, 64973, 18480, 38059, 20099] avg=32768.0000 max=64973 max_rank=4 min=7062 min_rank=0 imbalance_ratio=1.982819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[33096, 32282, 33096, 33097, 33096, 31286, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31286 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31876 max_load_reduction_pct=0.490604 moved_to_replicas=131976 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[40139, 46, 12254, 32790, 5855, 32303, 73248, 65509] avg=32768.0000 max=73248 max_rank=6 min=46 min_rank=1 imbalance_ratio=2.235352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[33096, 32282, 33096, 33097, 33096, 31286, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31286 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31876 max_load_reduction_pct=0.490604 moved_to_replicas=131976 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[33096, 32282, 33096, 33097, 33096, 31286, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31286 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31876 max_load_reduction_pct=0.490604 moved_to_replicas=131976 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[40139, 46, 12254, 32790, 5855, 32303, 73248, 65509] avg=32768.0000 max=73248 max_rank=6 min=46 min_rank=1 imbalance_ratio=2.235352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[40139, 46, 12254, 32790, 5855, 32303, 73248, 65509] avg=32768.0000 max=73248 max_rank=6 min=46 min_rank=1 imbalance_ratio=2.235352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[33096, 32282, 33096, 33097, 33096, 31286, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=31286 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31876 max_load_reduction_pct=0.490604 moved_to_replicas=131976 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[40139, 46, 12254, 32790, 5855, 32303, 73248, 65509] avg=32768.0000 max=73248 max_rank=6 min=46 min_rank=1 imbalance_ratio=2.235352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[33097, 32782, 33096, 32790, 31884, 32303, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31884 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40151 max_load_reduction_pct=0.548151 moved_to_replicas=159841 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[40139, 46, 12254, 32790, 5855, 32303, 73248, 65509] avg=32768.0000 max=73248 max_rank=6 min=46 min_rank=1 imbalance_ratio=2.235352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[33097, 32782, 33096, 32790, 31884, 32303, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31884 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40151 max_load_reduction_pct=0.548151 moved_to_replicas=159841 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[33097, 32782, 33096, 32790, 31884, 32303, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31884 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40151 max_load_reduction_pct=0.548151 moved_to_replicas=159841 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[40139, 46, 12254, 32790, 5855, 32303, 73248, 65509] avg=32768.0000 max=73248 max_rank=6 min=46 min_rank=1 imbalance_ratio=2.235352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[33097, 32782, 33096, 32790, 31884, 32303, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31884 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40151 max_load_reduction_pct=0.548151 moved_to_replicas=159841 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[40139, 46, 12254, 32790, 5855, 32303, 73248, 65509] avg=32768.0000 max=73248 max_rank=6 min=46 min_rank=1 imbalance_ratio=2.235352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[40139, 46, 12254, 32790, 5855, 32303, 73248, 65509] avg=32768.0000 max=73248 max_rank=6 min=46 min_rank=1 imbalance_ratio=2.235352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[33097, 32782, 33096, 32790, 31884, 32303, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31884 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40151 max_load_reduction_pct=0.548151 moved_to_replicas=159841 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[65528, 76000, 22301, 32769, 10, 65517, 15, 4] avg=32768.0000 max=76000 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.319336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[33097, 32782, 33096, 32790, 31884, 32303, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31884 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40151 max_load_reduction_pct=0.548151 moved_to_replicas=159841 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[65528, 76000, 22301, 32769, 10, 65517, 15, 4] avg=32768.0000 max=76000 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.319336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[33097, 32782, 33096, 32790, 31884, 32303, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31884 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40151 max_load_reduction_pct=0.548151 moved_to_replicas=159841 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[65528, 76000, 22301, 32769, 10, 65517, 15, 4] avg=32768.0000 max=76000 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.319336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[33097, 32782, 33096, 32790, 31884, 32303, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31884 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=40151 max_load_reduction_pct=0.548151 moved_to_replicas=159841 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[65528, 76000, 22301, 32769, 10, 65517, 15, 4] avg=32768.0000 max=76000 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.319336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33095, 31799, 32769, 33098, 33096, 32447, 32744] avg=32768.0000 max=33098 max_rank=4 min=31799 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=42902 max_load_reduction_pct=0.564500 moved_to_replicas=217183 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33095, 31799, 32769, 33098, 33096, 32447, 32744] avg=32768.0000 max=33098 max_rank=4 min=31799 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=42902 max_load_reduction_pct=0.564500 moved_to_replicas=217183 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[65528, 76000, 22301, 32769, 10, 65517, 15, 4] avg=32768.0000 max=76000 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.319336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33095, 31799, 32769, 33098, 33096, 32447, 32744] avg=32768.0000 max=33098 max_rank=4 min=31799 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=42902 max_load_reduction_pct=0.564500 moved_to_replicas=217183 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[65528, 76000, 22301, 32769, 10, 65517, 15, 4] avg=32768.0000 max=76000 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.319336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33095, 31799, 32769, 33098, 33096, 32447, 32744] avg=32768.0000 max=33098 max_rank=4 min=31799 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=42902 max_load_reduction_pct=0.564500 moved_to_replicas=217183 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[65528, 76000, 22301, 32769, 10, 65517, 15, 4] avg=32768.0000 max=76000 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.319336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[65528, 76000, 22301, 32769, 10, 65517, 15, 4] avg=32768.0000 max=76000 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.319336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[65531, 32769, 32769, 9, 32834, 34919, 584, 62729] avg=32768.0000 max=65531 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33095, 31799, 32769, 33098, 33096, 32447, 32744] avg=32768.0000 max=33098 max_rank=4 min=31799 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=42902 max_load_reduction_pct=0.564500 moved_to_replicas=217183 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33095, 31799, 32769, 33098, 33096, 32447, 32744] avg=32768.0000 max=33098 max_rank=4 min=31799 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=42902 max_load_reduction_pct=0.564500 moved_to_replicas=217183 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[65531, 32769, 32769, 9, 32834, 34919, 584, 62729] avg=32768.0000 max=65531 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[65531, 32769, 32769, 9, 32834, 34919, 584, 62729] avg=32768.0000 max=65531 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33095, 31799, 32769, 33098, 33096, 32447, 32744] avg=32768.0000 max=33098 max_rank=4 min=31799 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=42902 max_load_reduction_pct=0.564500 moved_to_replicas=217183 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33095, 31799, 32769, 33098, 33096, 32447, 32744] avg=32768.0000 max=33098 max_rank=4 min=31799 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=42902 max_load_reduction_pct=0.564500 moved_to_replicas=217183 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[65531, 32769, 32769, 9, 32834, 34919, 584, 62729] avg=32768.0000 max=65531 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[33096, 32769, 32769, 32444, 32834, 33096, 32040, 33096] avg=32768.0000 max=33096 max_rank=0 min=32040 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=127751 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[33096, 32769, 32769, 32444, 32834, 33096, 32040, 33096] avg=32768.0000 max=33096 max_rank=0 min=32040 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=127751 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[33096, 32769, 32769, 32444, 32834, 33096, 32040, 33096] avg=32768.0000 max=33096 max_rank=0 min=32040 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=127751 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[65531, 32769, 32769, 9, 32834, 34919, 584, 62729] avg=32768.0000 max=65531 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[33096, 32769, 32769, 32444, 32834, 33096, 32040, 33096] avg=32768.0000 max=33096 max_rank=0 min=32040 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=127751 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[65531, 32769, 32769, 9, 32834, 34919, 584, 62729] avg=32768.0000 max=65531 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[65531, 32769, 32769, 9, 32834, 34919, 584, 62729] avg=32768.0000 max=65531 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[28, 63108, 35190, 14, 65528, 32765, 65506, 5] avg=32768.0000 max=65528 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[65531, 32769, 32769, 9, 32834, 34919, 584, 62729] avg=32768.0000 max=65531 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[33096, 32769, 32769, 32444, 32834, 33096, 32040, 33096] avg=32768.0000 max=33096 max_rank=0 min=32040 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=127751 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[33096, 32769, 32769, 32444, 32834, 33096, 32040, 33096] avg=32768.0000 max=33096 max_rank=0 min=32040 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=127751 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[28, 63108, 35190, 14, 65528, 32765, 65506, 5] avg=32768.0000 max=65528 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[33096, 32769, 32769, 32444, 32834, 33096, 32040, 33096] avg=32768.0000 max=33096 max_rank=0 min=32040 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=127751 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[28, 63108, 35190, 14, 65528, 32765, 65506, 5] avg=32768.0000 max=65528 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[33096, 32769, 32769, 32444, 32834, 33096, 32040, 33096] avg=32768.0000 max=33096 max_rank=0 min=32040 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=127751 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[32134, 33096, 33096, 32424, 33096, 32765, 33096, 32437] avg=32768.0000 max=33096 max_rank=1 min=32134 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=193829 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[28, 63108, 35190, 14, 65528, 32765, 65506, 5] avg=32768.0000 max=65528 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[32134, 33096, 33096, 32424, 33096, 32765, 33096, 32437] avg=32768.0000 max=33096 max_rank=1 min=32134 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=193829 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[28, 63108, 35190, 14, 65528, 32765, 65506, 5] avg=32768.0000 max=65528 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[32134, 33096, 33096, 32424, 33096, 32765, 33096, 32437] avg=32768.0000 max=33096 max_rank=1 min=32134 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=193829 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[28, 63108, 35190, 14, 65528, 32765, 65506, 5] avg=32768.0000 max=65528 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[32134, 33096, 33096, 32424, 33096, 32765, 33096, 32437] avg=32768.0000 max=33096 max_rank=1 min=32134 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=193829 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[28, 63108, 35190, 14, 65528, 32765, 65506, 5] avg=32768.0000 max=65528 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[28, 63108, 35190, 14, 65528, 32765, 65506, 5] avg=32768.0000 max=65528 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[98165, 12, 32816, 32833, 14, 65529, 8, 32767] avg=32768.0000 max=98165 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.995758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[32134, 33096, 33096, 32424, 33096, 32765, 33096, 32437] avg=32768.0000 max=33096 max_rank=1 min=32134 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=193829 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[32134, 33096, 33096, 32424, 33096, 32765, 33096, 32437] avg=32768.0000 max=33096 max_rank=1 min=32134 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=193829 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[98165, 12, 32816, 32833, 14, 65529, 8, 32767] avg=32768.0000 max=98165 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.995758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[32134, 33096, 33096, 32424, 33096, 32765, 33096, 32437] avg=32768.0000 max=33096 max_rank=1 min=32134 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=193829 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[98165, 12, 32816, 32833, 14, 65529, 8, 32767] avg=32768.0000 max=98165 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.995758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32320, 32816, 32833, 32447, 33096, 32769, 32767] avg=32768.0000 max=33096 max_rank=0 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65069 max_load_reduction_pct=0.662853 moved_to_replicas=194889 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[32134, 33096, 33096, 32424, 33096, 32765, 33096, 32437] avg=32768.0000 max=33096 max_rank=1 min=32134 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=193829 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[98165, 12, 32816, 32833, 14, 65529, 8, 32767] avg=32768.0000 max=98165 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.995758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32320, 32816, 32833, 32447, 33096, 32769, 32767] avg=32768.0000 max=33096 max_rank=0 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65069 max_load_reduction_pct=0.662853 moved_to_replicas=194889 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32320, 32816, 32833, 32447, 33096, 32769, 32767] avg=32768.0000 max=33096 max_rank=0 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65069 max_load_reduction_pct=0.662853 moved_to_replicas=194889 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[98165, 12, 32816, 32833, 14, 65529, 8, 32767] avg=32768.0000 max=98165 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.995758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[98165, 12, 32816, 32833, 14, 65529, 8, 32767] avg=32768.0000 max=98165 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.995758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32320, 32816, 32833, 32447, 33096, 32769, 32767] avg=32768.0000 max=33096 max_rank=0 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65069 max_load_reduction_pct=0.662853 moved_to_replicas=194889 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[98165, 12, 32816, 32833, 14, 65529, 8, 32767] avg=32768.0000 max=98165 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.995758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[6, 28, 65521, 32760, 65522, 32772, 32763, 32772] avg=32768.0000 max=65522 max_rank=4 min=6 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[98165, 12, 32816, 32833, 14, 65529, 8, 32767] avg=32768.0000 max=98165 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.995758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32320, 32816, 32833, 32447, 33096, 32769, 32767] avg=32768.0000 max=33096 max_rank=0 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65069 max_load_reduction_pct=0.662853 moved_to_replicas=194889 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32320, 32816, 32833, 32447, 33096, 32769, 32767] avg=32768.0000 max=33096 max_rank=0 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65069 max_load_reduction_pct=0.662853 moved_to_replicas=194889 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[6, 28, 65521, 32760, 65522, 32772, 32763, 32772] avg=32768.0000 max=65522 max_rank=4 min=6 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[6, 28, 65521, 32760, 65522, 32772, 32763, 32772] avg=32768.0000 max=65522 max_rank=4 min=6 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32320, 32816, 32833, 32447, 33096, 32769, 32767] avg=32768.0000 max=33096 max_rank=0 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65069 max_load_reduction_pct=0.662853 moved_to_replicas=194889 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[32432, 32453, 33096, 32760, 33096, 32772, 32763, 32772] avg=32768.0000 max=33096 max_rank=2 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129681 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[6, 28, 65521, 32760, 65522, 32772, 32763, 32772] avg=32768.0000 max=65522 max_rank=4 min=6 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32320, 32816, 32833, 32447, 33096, 32769, 32767] avg=32768.0000 max=33096 max_rank=0 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65069 max_load_reduction_pct=0.662853 moved_to_replicas=194889 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[32432, 32453, 33096, 32760, 33096, 32772, 32763, 32772] avg=32768.0000 max=33096 max_rank=2 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129681 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[32432, 32453, 33096, 32760, 33096, 32772, 32763, 32772] avg=32768.0000 max=33096 max_rank=2 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129681 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[6, 28, 65521, 32760, 65522, 32772, 32763, 32772] avg=32768.0000 max=65522 max_rank=4 min=6 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[6, 28, 65521, 32760, 65522, 32772, 32763, 32772] avg=32768.0000 max=65522 max_rank=4 min=6 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[32432, 32453, 33096, 32760, 33096, 32772, 32763, 32772] avg=32768.0000 max=33096 max_rank=2 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129681 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[18, 65525, 32684, 65510, 32766, 32870, 32763, 8] avg=32768.0000 max=65525 max_rank=1 min=8 min_rank=7 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[6, 28, 65521, 32760, 65522, 32772, 32763, 32772] avg=32768.0000 max=65522 max_rank=4 min=6 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[6, 28, 65521, 32760, 65522, 32772, 32763, 32772] avg=32768.0000 max=65522 max_rank=4 min=6 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[32432, 32453, 33096, 32760, 33096, 32772, 32763, 32772] avg=32768.0000 max=33096 max_rank=2 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129681 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[18, 65525, 32684, 65510, 32766, 32870, 32763, 8] avg=32768.0000 max=65525 max_rank=1 min=8 min_rank=7 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[32432, 32453, 33096, 32760, 33096, 32772, 32763, 32772] avg=32768.0000 max=33096 max_rank=2 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129681 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[18, 65525, 32684, 65510, 32766, 32870, 32763, 8] avg=32768.0000 max=65525 max_rank=1 min=8 min_rank=7 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[32432, 33096, 32684, 33096, 32766, 32870, 32763, 32437] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=129630 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[18, 65525, 32684, 65510, 32766, 32870, 32763, 8] avg=32768.0000 max=65525 max_rank=1 min=8 min_rank=7 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[32432, 32453, 33096, 32760, 33096, 32772, 32763, 32772] avg=32768.0000 max=33096 max_rank=2 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129681 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[32432, 32453, 33096, 32760, 33096, 32772, 32763, 32772] avg=32768.0000 max=33096 max_rank=2 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129681 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[32432, 33096, 32684, 33096, 32766, 32870, 32763, 32437] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=129630 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[32432, 33096, 32684, 33096, 32766, 32870, 32763, 32437] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=129630 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[18, 65525, 32684, 65510, 32766, 32870, 32763, 8] avg=32768.0000 max=65525 max_rank=1 min=8 min_rank=7 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[18, 65525, 32684, 65510, 32766, 32870, 32763, 8] avg=32768.0000 max=65525 max_rank=1 min=8 min_rank=7 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[32432, 33096, 32684, 33096, 32766, 32870, 32763, 32437] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=129630 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[65514, 31356, 67021, 18, 18, 35, 32670, 65512] avg=32768.0000 max=67021 max_rank=2 min=18 min_rank=3 imbalance_ratio=2.045319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[18, 65525, 32684, 65510, 32766, 32870, 32763, 8] avg=32768.0000 max=65525 max_rank=1 min=8 min_rank=7 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[18, 65525, 32684, 65510, 32766, 32870, 32763, 8] avg=32768.0000 max=65525 max_rank=1 min=8 min_rank=7 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[65514, 31356, 67021, 18, 18, 35, 32670, 65512] avg=32768.0000 max=67021 max_rank=2 min=18 min_rank=3 imbalance_ratio=2.045319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[32432, 33096, 32684, 33096, 32766, 32870, 32763, 32437] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=129630 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[32432, 33096, 32684, 33096, 32766, 32870, 32763, 32437] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=129630 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[65514, 31356, 67021, 18, 18, 35, 32670, 65512] avg=32768.0000 max=67021 max_rank=2 min=18 min_rank=3 imbalance_ratio=2.045319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[33096, 31860, 33096, 32777, 33096, 32453, 32670, 33096] avg=32768.0000 max=33096 max_rank=0 min=31860 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33925 max_load_reduction_pct=0.506185 moved_to_replicas=197563 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[65514, 31356, 67021, 18, 18, 35, 32670, 65512] avg=32768.0000 max=67021 max_rank=2 min=18 min_rank=3 imbalance_ratio=2.045319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[32432, 33096, 32684, 33096, 32766, 32870, 32763, 32437] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=129630 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[32432, 33096, 32684, 33096, 32766, 32870, 32763, 32437] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=129630 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[33096, 31860, 33096, 32777, 33096, 32453, 32670, 33096] avg=32768.0000 max=33096 max_rank=0 min=31860 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33925 max_load_reduction_pct=0.506185 moved_to_replicas=197563 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[33096, 31860, 33096, 32777, 33096, 32453, 32670, 33096] avg=32768.0000 max=33096 max_rank=0 min=31860 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33925 max_load_reduction_pct=0.506185 moved_to_replicas=197563 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[65514, 31356, 67021, 18, 18, 35, 32670, 65512] avg=32768.0000 max=67021 max_rank=2 min=18 min_rank=3 imbalance_ratio=2.045319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[65514, 31356, 67021, 18, 18, 35, 32670, 65512] avg=32768.0000 max=67021 max_rank=2 min=18 min_rank=3 imbalance_ratio=2.045319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[33096, 31860, 33096, 32777, 33096, 32453, 32670, 33096] avg=32768.0000 max=33096 max_rank=0 min=31860 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33925 max_load_reduction_pct=0.506185 moved_to_replicas=197563 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[33309, 65522, 63781, 33917, 12, 32799, 32794, 10] avg=32768.0000 max=65522 max_rank=1 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[33309, 65522, 63781, 33917, 12, 32799, 32794, 10] avg=32768.0000 max=65522 max_rank=1 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[65514, 31356, 67021, 18, 18, 35, 32670, 65512] avg=32768.0000 max=67021 max_rank=2 min=18 min_rank=3 imbalance_ratio=2.045319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[65514, 31356, 67021, 18, 18, 35, 32670, 65512] avg=32768.0000 max=67021 max_rank=2 min=18 min_rank=3 imbalance_ratio=2.045319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[33309, 65522, 63781, 33917, 12, 32799, 32794, 10] avg=32768.0000 max=65522 max_rank=1 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[33096, 31860, 33096, 32777, 33096, 32453, 32670, 33096] avg=32768.0000 max=33096 max_rank=0 min=31860 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33925 max_load_reduction_pct=0.506185 moved_to_replicas=197563 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[33096, 31860, 33096, 32777, 33096, 32453, 32670, 33096] avg=32768.0000 max=33096 max_rank=0 min=31860 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33925 max_load_reduction_pct=0.506185 moved_to_replicas=197563 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31731, 32799, 32794, 32436] avg=32768.0000 max=33096 max_rank=0 min=31731 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128272 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[33309, 65522, 63781, 33917, 12, 32799, 32794, 10] avg=32768.0000 max=65522 max_rank=1 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31731, 32799, 32794, 32436] avg=32768.0000 max=33096 max_rank=0 min=31731 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128272 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[33096, 31860, 33096, 32777, 33096, 32453, 32670, 33096] avg=32768.0000 max=33096 max_rank=0 min=31860 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33925 max_load_reduction_pct=0.506185 moved_to_replicas=197563 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31731, 32799, 32794, 32436] avg=32768.0000 max=33096 max_rank=0 min=31731 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128272 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[33096, 31860, 33096, 32777, 33096, 32453, 32670, 33096] avg=32768.0000 max=33096 max_rank=0 min=31860 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33925 max_load_reduction_pct=0.506185 moved_to_replicas=197563 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31731, 32799, 32794, 32436] avg=32768.0000 max=33096 max_rank=0 min=31731 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128272 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[33309, 65522, 63781, 33917, 12, 32799, 32794, 10] avg=32768.0000 max=65522 max_rank=1 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[32775, 6, 32773, 32767, 34460, 63817, 32777, 32769] avg=32768.0000 max=63817 max_rank=5 min=6 min_rank=1 imbalance_ratio=1.947540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[33309, 65522, 63781, 33917, 12, 32799, 32794, 10] avg=32768.0000 max=65522 max_rank=1 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[32775, 6, 32773, 32767, 34460, 63817, 32777, 32769] avg=32768.0000 max=63817 max_rank=5 min=6 min_rank=1 imbalance_ratio=1.947540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[33309, 65522, 63781, 33917, 12, 32799, 32794, 10] avg=32768.0000 max=65522 max_rank=1 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[32775, 6, 32773, 32767, 34460, 63817, 32777, 32769] avg=32768.0000 max=63817 max_rank=5 min=6 min_rank=1 imbalance_ratio=1.947540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31731, 32799, 32794, 32436] avg=32768.0000 max=33096 max_rank=0 min=31731 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128272 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[32775, 32091, 32773, 32767, 33096, 33096, 32777, 32769] avg=32768.0000 max=33096 max_rank=4 min=32091 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30721 max_load_reduction_pct=0.481392 moved_to_replicas=64153 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[33309, 65522, 63781, 33917, 12, 32799, 32794, 10] avg=32768.0000 max=65522 max_rank=1 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31731, 32799, 32794, 32436] avg=32768.0000 max=33096 max_rank=0 min=31731 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128272 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[32775, 6, 32773, 32767, 34460, 63817, 32777, 32769] avg=32768.0000 max=63817 max_rank=5 min=6 min_rank=1 imbalance_ratio=1.947540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[32775, 32091, 32773, 32767, 33096, 33096, 32777, 32769] avg=32768.0000 max=33096 max_rank=4 min=32091 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30721 max_load_reduction_pct=0.481392 moved_to_replicas=64153 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31731, 32799, 32794, 32436] avg=32768.0000 max=33096 max_rank=0 min=31731 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128272 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[32775, 32091, 32773, 32767, 33096, 33096, 32777, 32769] avg=32768.0000 max=33096 max_rank=4 min=32091 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30721 max_load_reduction_pct=0.481392 moved_to_replicas=64153 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31731, 32799, 32794, 32436] avg=32768.0000 max=33096 max_rank=0 min=31731 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128272 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[32775, 32091, 32773, 32767, 33096, 33096, 32777, 32769] avg=32768.0000 max=33096 max_rank=4 min=32091 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30721 max_load_reduction_pct=0.481392 moved_to_replicas=64153 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[32775, 6, 32773, 32767, 34460, 63817, 32777, 32769] avg=32768.0000 max=63817 max_rank=5 min=6 min_rank=1 imbalance_ratio=1.947540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[65521, 97427, 32771, 16, 40, 32773, 32767, 829] avg=32768.0000 max=97427 max_rank=1 min=16 min_rank=3 imbalance_ratio=2.973236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[32775, 6, 32773, 32767, 34460, 63817, 32777, 32769] avg=32768.0000 max=63817 max_rank=5 min=6 min_rank=1 imbalance_ratio=1.947540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[65521, 97427, 32771, 16, 40, 32773, 32767, 829] avg=32768.0000 max=97427 max_rank=1 min=16 min_rank=3 imbalance_ratio=2.973236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[65521, 97427, 32771, 16, 40, 32773, 32767, 829] avg=32768.0000 max=97427 max_rank=1 min=16 min_rank=3 imbalance_ratio=2.973236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[32775, 6, 32773, 32767, 34460, 63817, 32777, 32769] avg=32768.0000 max=63817 max_rank=5 min=6 min_rank=1 imbalance_ratio=1.947540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[32775, 32091, 32773, 32767, 33096, 33096, 32777, 32769] avg=32768.0000 max=33096 max_rank=4 min=32091 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30721 max_load_reduction_pct=0.481392 moved_to_replicas=64153 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 32771, 32716, 31829, 32773, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64331 max_load_reduction_pct=0.660300 moved_to_replicas=191521 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[32775, 6, 32773, 32767, 34460, 63817, 32777, 32769] avg=32768.0000 max=63817 max_rank=5 min=6 min_rank=1 imbalance_ratio=1.947540 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[32775, 32091, 32773, 32767, 33096, 33096, 32777, 32769] avg=32768.0000 max=33096 max_rank=4 min=32091 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30721 max_load_reduction_pct=0.481392 moved_to_replicas=64153 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[65521, 97427, 32771, 16, 40, 32773, 32767, 829] avg=32768.0000 max=97427 max_rank=1 min=16 min_rank=3 imbalance_ratio=2.973236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 32771, 32716, 31829, 32773, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64331 max_load_reduction_pct=0.660300 moved_to_replicas=191521 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 32771, 32716, 31829, 32773, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64331 max_load_reduction_pct=0.660300 moved_to_replicas=191521 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[32775, 32091, 32773, 32767, 33096, 33096, 32777, 32769] avg=32768.0000 max=33096 max_rank=4 min=32091 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30721 max_load_reduction_pct=0.481392 moved_to_replicas=64153 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[32775, 32091, 32773, 32767, 33096, 33096, 32777, 32769] avg=32768.0000 max=33096 max_rank=4 min=32091 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30721 max_load_reduction_pct=0.481392 moved_to_replicas=64153 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 32771, 32716, 31829, 32773, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64331 max_load_reduction_pct=0.660300 moved_to_replicas=191521 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[35, 131024, 9, 14, 64870, 32778, 646, 32768] avg=32768.0000 max=131024 max_rank=1 min=9 min_rank=2 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[65521, 97427, 32771, 16, 40, 32773, 32767, 829] avg=32768.0000 max=97427 max_rank=1 min=16 min_rank=3 imbalance_ratio=2.973236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[65521, 97427, 32771, 16, 40, 32773, 32767, 829] avg=32768.0000 max=97427 max_rank=1 min=16 min_rank=3 imbalance_ratio=2.973236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[35, 131024, 9, 14, 64870, 32778, 646, 32768] avg=32768.0000 max=131024 max_rank=1 min=9 min_rank=2 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[35, 131024, 9, 14, 64870, 32778, 646, 32768] avg=32768.0000 max=131024 max_rank=1 min=9 min_rank=2 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[65521, 97427, 32771, 16, 40, 32773, 32767, 829] avg=32768.0000 max=97427 max_rank=1 min=16 min_rank=3 imbalance_ratio=2.973236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[32443, 33096, 32769, 32774, 33096, 32778, 32420, 32768] avg=32768.0000 max=33096 max_rank=1 min=32420 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97928 max_load_reduction_pct=0.747405 moved_to_replicas=258872 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 32771, 32716, 31829, 32773, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64331 max_load_reduction_pct=0.660300 moved_to_replicas=191521 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 32771, 32716, 31829, 32773, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64331 max_load_reduction_pct=0.660300 moved_to_replicas=191521 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[65521, 97427, 32771, 16, 40, 32773, 32767, 829] avg=32768.0000 max=97427 max_rank=1 min=16 min_rank=3 imbalance_ratio=2.973236 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[32443, 33096, 32769, 32774, 33096, 32778, 32420, 32768] avg=32768.0000 max=33096 max_rank=1 min=32420 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97928 max_load_reduction_pct=0.747405 moved_to_replicas=258872 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[35, 131024, 9, 14, 64870, 32778, 646, 32768] avg=32768.0000 max=131024 max_rank=1 min=9 min_rank=2 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[32443, 33096, 32769, 32774, 33096, 32778, 32420, 32768] avg=32768.0000 max=33096 max_rank=1 min=32420 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97928 max_load_reduction_pct=0.747405 moved_to_replicas=258872 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 32771, 32716, 31829, 32773, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64331 max_load_reduction_pct=0.660300 moved_to_replicas=191521 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 32771, 32716, 31829, 32773, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64331 max_load_reduction_pct=0.660300 moved_to_replicas=191521 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[32443, 33096, 32769, 32774, 33096, 32778, 32420, 32768] avg=32768.0000 max=33096 max_rank=1 min=32420 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97928 max_load_reduction_pct=0.747405 moved_to_replicas=258872 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[13, 32776, 65541, 32764, 8, 32775, 32745, 65522] avg=32768.0000 max=65541 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[35, 131024, 9, 14, 64870, 32778, 646, 32768] avg=32768.0000 max=131024 max_rank=1 min=9 min_rank=2 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[35, 131024, 9, 14, 64870, 32778, 646, 32768] avg=32768.0000 max=131024 max_rank=1 min=9 min_rank=2 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[13, 32776, 65541, 32764, 8, 32775, 32745, 65522] avg=32768.0000 max=65541 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[13, 32776, 65541, 32764, 8, 32775, 32745, 65522] avg=32768.0000 max=65541 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[35, 131024, 9, 14, 64870, 32778, 646, 32768] avg=32768.0000 max=131024 max_rank=1 min=9 min_rank=2 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[32439, 32776, 33096, 32764, 32453, 32775, 32745, 33096] avg=32768.0000 max=33096 max_rank=2 min=32439 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129732 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[32443, 33096, 32769, 32774, 33096, 32778, 32420, 32768] avg=32768.0000 max=33096 max_rank=1 min=32420 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97928 max_load_reduction_pct=0.747405 moved_to_replicas=258872 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[35, 131024, 9, 14, 64870, 32778, 646, 32768] avg=32768.0000 max=131024 max_rank=1 min=9 min_rank=2 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[32443, 33096, 32769, 32774, 33096, 32778, 32420, 32768] avg=32768.0000 max=33096 max_rank=1 min=32420 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97928 max_load_reduction_pct=0.747405 moved_to_replicas=258872 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[32439, 32776, 33096, 32764, 32453, 32775, 32745, 33096] avg=32768.0000 max=33096 max_rank=2 min=32439 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129732 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[32439, 32776, 33096, 32764, 32453, 32775, 32745, 33096] avg=32768.0000 max=33096 max_rank=2 min=32439 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129732 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[13, 32776, 65541, 32764, 8, 32775, 32745, 65522] avg=32768.0000 max=65541 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[32443, 33096, 32769, 32774, 33096, 32778, 32420, 32768] avg=32768.0000 max=33096 max_rank=1 min=32420 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97928 max_load_reduction_pct=0.747405 moved_to_replicas=258872 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[32443, 33096, 32769, 32774, 33096, 32778, 32420, 32768] avg=32768.0000 max=33096 max_rank=1 min=32420 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97928 max_load_reduction_pct=0.747405 moved_to_replicas=258872 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32754, 164, 20, 38, 65524, 98238, 32632] avg=32768.0000 max=98238 max_rank=6 min=20 min_rank=3 imbalance_ratio=2.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[32439, 32776, 33096, 32764, 32453, 32775, 32745, 33096] avg=32768.0000 max=33096 max_rank=2 min=32439 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129732 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[13, 32776, 65541, 32764, 8, 32775, 32745, 65522] avg=32768.0000 max=65541 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32754, 164, 20, 38, 65524, 98238, 32632] avg=32768.0000 max=98238 max_rank=6 min=20 min_rank=3 imbalance_ratio=2.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[13, 32776, 65541, 32764, 8, 32775, 32745, 65522] avg=32768.0000 max=65541 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32754, 164, 20, 38, 65524, 98238, 32632] avg=32768.0000 max=98238 max_rank=6 min=20 min_rank=3 imbalance_ratio=2.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[13, 32776, 65541, 32764, 8, 32775, 32745, 65522] avg=32768.0000 max=65541 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32754, 32592, 32776, 32424, 33096, 33096, 32632] avg=32768.0000 max=33096 max_rank=5 min=32424 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65142 max_load_reduction_pct=0.663104 moved_to_replicas=195116 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[32439, 32776, 33096, 32764, 32453, 32775, 32745, 33096] avg=32768.0000 max=33096 max_rank=2 min=32439 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129732 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32754, 32592, 32776, 32424, 33096, 33096, 32632] avg=32768.0000 max=33096 max_rank=5 min=32424 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65142 max_load_reduction_pct=0.663104 moved_to_replicas=195116 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32754, 32592, 32776, 32424, 33096, 33096, 32632] avg=32768.0000 max=33096 max_rank=5 min=32424 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65142 max_load_reduction_pct=0.663104 moved_to_replicas=195116 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[32439, 32776, 33096, 32764, 32453, 32775, 32745, 33096] avg=32768.0000 max=33096 max_rank=2 min=32439 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129732 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32754, 164, 20, 38, 65524, 98238, 32632] avg=32768.0000 max=98238 max_rank=6 min=20 min_rank=3 imbalance_ratio=2.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[13, 32776, 65541, 32764, 8, 32775, 32745, 65522] avg=32768.0000 max=65541 max_rank=2 min=8 min_rank=4 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[32439, 32776, 33096, 32764, 32453, 32775, 32745, 33096] avg=32768.0000 max=33096 max_rank=2 min=32439 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129732 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[65242, 123, 32728, 17, 32757, 244, 65519, 65514] avg=32768.0000 max=65519 max_rank=6 min=17 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32754, 164, 20, 38, 65524, 98238, 32632] avg=32768.0000 max=98238 max_rank=6 min=20 min_rank=3 imbalance_ratio=2.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32754, 32592, 32776, 32424, 33096, 33096, 32632] avg=32768.0000 max=33096 max_rank=5 min=32424 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65142 max_load_reduction_pct=0.663104 moved_to_replicas=195116 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[32439, 32776, 33096, 32764, 32453, 32775, 32745, 33096] avg=32768.0000 max=33096 max_rank=2 min=32439 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129732 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[65242, 123, 32728, 17, 32757, 244, 65519, 65514] avg=32768.0000 max=65519 max_rank=6 min=17 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[65242, 123, 32728, 17, 32757, 244, 65519, 65514] avg=32768.0000 max=65519 max_rank=6 min=17 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32754, 164, 20, 38, 65524, 98238, 32632] avg=32768.0000 max=98238 max_rank=6 min=20 min_rank=3 imbalance_ratio=2.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32754, 164, 20, 38, 65524, 98238, 32632] avg=32768.0000 max=98238 max_rank=6 min=20 min_rank=3 imbalance_ratio=2.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[33096, 32541, 32728, 32440, 32757, 32390, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=193975 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32754, 32592, 32776, 32424, 33096, 33096, 32632] avg=32768.0000 max=33096 max_rank=5 min=32424 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65142 max_load_reduction_pct=0.663104 moved_to_replicas=195116 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[33096, 32541, 32728, 32440, 32757, 32390, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=193975 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[33096, 32541, 32728, 32440, 32757, 32390, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=193975 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32754, 32592, 32776, 32424, 33096, 33096, 32632] avg=32768.0000 max=33096 max_rank=5 min=32424 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65142 max_load_reduction_pct=0.663104 moved_to_replicas=195116 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[65242, 123, 32728, 17, 32757, 244, 65519, 65514] avg=32768.0000 max=65519 max_rank=6 min=17 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32754, 164, 20, 38, 65524, 98238, 32632] avg=32768.0000 max=98238 max_rank=6 min=20 min_rank=3 imbalance_ratio=2.997986 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32754, 32592, 32776, 32424, 33096, 33096, 32632] avg=32768.0000 max=33096 max_rank=5 min=32424 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65142 max_load_reduction_pct=0.663104 moved_to_replicas=195116 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[65242, 123, 32728, 17, 32757, 244, 65519, 65514] avg=32768.0000 max=65519 max_rank=6 min=17 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[33096, 32541, 32728, 32440, 32757, 32390, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=193975 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32754, 32592, 32776, 32424, 33096, 33096, 32632] avg=32768.0000 max=33096 max_rank=5 min=32424 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65142 max_load_reduction_pct=0.663104 moved_to_replicas=195116 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[65242, 123, 32728, 17, 32757, 244, 65519, 65514] avg=32768.0000 max=65519 max_rank=6 min=17 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[65242, 123, 32728, 17, 32757, 244, 65519, 65514] avg=32768.0000 max=65519 max_rank=6 min=17 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[33096, 32541, 32728, 32440, 32757, 32390, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=193975 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[33096, 32541, 32728, 32440, 32757, 32390, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=193975 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[65242, 123, 32728, 17, 32757, 244, 65519, 65514] avg=32768.0000 max=65519 max_rank=6 min=17 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[33096, 32541, 32728, 32440, 32757, 32390, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=193975 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[33096, 32541, 32728, 32440, 32757, 32390, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32390 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=193975 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=32 total_expected_recv=1050620 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=51840 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=32 total_expected_recv=1047843 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56192 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=32 total_expected_recv=1046649 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=55424 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=32 total_expected_recv=1047438 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56320 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=32 total_expected_recv=1048726 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=54528 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=32 total_expected_recv=1050578 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=53120 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=32 total_expected_recv=1049012 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=54912 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=32 total_expected_recv=1047742 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56832 peak_demand=33096 required_capacity=1.010010 + [2026-08-30 05:11:44] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 549.2 | throughput per GPU (TFLOP/s/GPU): 607.0 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 1.013027E+01 | loss scale: 1.0 | grad norm: 7.719 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=34816 expected_recv=65371 expected_overflow=0 exact_recv_per_rank=[32275, 31293, 33097, 33096, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=34816 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[32275, 31293, 33097, 33096, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32275, 31293, 33097, 33096, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=34816 expected_recv=64389 expected_overflow=0 exact_recv_per_rank=[32275, 31293, 33097, 33096, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32275, 31293, 33097, 33096, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32275, 31293, 33097, 33096, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32275, 31293, 33097, 33096, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=34816 expected_recv=65371 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=34816 expected_recv=66194 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=34816 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[32275, 31293, 33097, 33096, 33096, 33095, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=34816 expected_recv=64389 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=67456 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=34816 expected_recv=63568 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=64512 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 32753, 32810, 32759, 32771, 32095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=34816 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 32753, 32810, 32759, 32771, 32095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=34816 expected_recv=65599 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 32753, 32810, 32759, 32771, 32095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=34816 expected_recv=65523 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=34816 expected_recv=65599 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=34816 expected_recv=65528 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 32753, 32810, 32759, 32771, 32095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=34816 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 32753, 32810, 32759, 32771, 32095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 32753, 32810, 32759, 32771, 32095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=34816 expected_recv=65528 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=34816 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 32753, 32810, 32759, 32771, 32095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=34816 expected_recv=65535 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=34816 expected_recv=64187 expected_overflow=0 exact_recv_per_rank=[33096, 32764, 32753, 32810, 32759, 32771, 32095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=34816 expected_recv=65532 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=34816 expected_recv=64187 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 33096, 32771, 32772, 32365, 32738, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 33096, 32771, 32772, 32365, 32738, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=34816 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 33096, 32771, 32772, 32365, 32738, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=34816 expected_recv=65540 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=34816 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 33096, 32771, 32772, 32365, 32738, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=34816 expected_recv=64736 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 33096, 32771, 32772, 32365, 32738, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=34816 expected_recv=65064 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 33096, 32771, 32772, 32365, 32738, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=34816 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 33096, 32771, 32772, 32365, 32738, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=34816 expected_recv=65539 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=34816 expected_recv=65488 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 33096, 32771, 32772, 32365, 32738, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=34816 expected_recv=64736 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=34816 expected_recv=65064 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=34816 expected_recv=65537 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=34816 expected_recv=65396 expected_overflow=0 exact_recv_per_rank=[32673, 31479, 33096, 32769, 33096, 32841, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=34816 expected_recv=65488 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=34816 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[32673, 31479, 33096, 32769, 33096, 32841, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32673, 31479, 33096, 32769, 33096, 32841, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=34816 expected_recv=65396 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=34816 expected_recv=65536 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=34816 expected_recv=62945 expected_overflow=0 exact_recv_per_rank=[32673, 31479, 33096, 32769, 33096, 32841, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32673, 31479, 33096, 32769, 33096, 32841, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=34816 expected_recv=65644 expected_overflow=0 exact_recv_per_rank=[32673, 31479, 33096, 32769, 33096, 32841, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=34816 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[32673, 31479, 33096, 32769, 33096, 32841, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=34816 expected_recv=62945 actual_valid_recv=63744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=34816 expected_recv=65644 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32673, 31479, 33096, 32769, 33096, 32841, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=34816 expected_recv=64309 expected_overflow=0 exact_recv_per_rank=[32097, 32772, 33096, 32779, 32738, 32770, 32796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=34816 expected_recv=66190 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=63744 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=34816 expected_recv=65547 expected_overflow=0 exact_recv_per_rank=[32097, 32772, 33096, 32779, 32738, 32770, 32796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=34816 expected_recv=64309 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=34816 expected_recv=66080 expected_overflow=0 exact_recv_per_rank=[32097, 32772, 33096, 32779, 32738, 32770, 32796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=34816 expected_recv=65547 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=34816 expected_recv=66080 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=34816 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[32097, 32772, 33096, 32779, 32738, 32770, 32796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=34816 expected_recv=65502 expected_overflow=0 exact_recv_per_rank=[32097, 32772, 33096, 32779, 32738, 32770, 32796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=34816 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[32097, 32772, 33096, 32779, 32738, 32770, 32796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=34816 expected_recv=65539 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=34816 expected_recv=65502 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=34816 expected_recv=65542 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=34816 expected_recv=65454 expected_overflow=0 exact_recv_per_rank=[32724, 32850, 32760, 32768, 32411, 33096, 32767, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32097, 32772, 33096, 32779, 32738, 32770, 32796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=34816 expected_recv=65577 expected_overflow=0 exact_recv_per_rank=[32097, 32772, 33096, 32779, 32738, 32770, 32796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=34816 expected_recv=65454 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=34816 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[32724, 32850, 32760, 32768, 32411, 33096, 32767, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=34816 expected_recv=65577 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=34816 expected_recv=65519 expected_overflow=0 exact_recv_per_rank=[32724, 32850, 32760, 32768, 32411, 33096, 32767, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=34816 expected_recv=65540 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=34816 expected_recv=65519 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32724, 32850, 32760, 32768, 32411, 33096, 32767, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=34816 expected_recv=64846 expected_overflow=0 exact_recv_per_rank=[32724, 32850, 32760, 32768, 32411, 33096, 32767, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=34816 expected_recv=65657 expected_overflow=0 exact_recv_per_rank=[32724, 32850, 32760, 32768, 32411, 33096, 32767, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=34816 expected_recv=64846 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 32247, 33098, 33096, 32779, 32305, 32429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=34816 expected_recv=65657 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=34816 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[32724, 32850, 32760, 32768, 32411, 33096, 32767, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=34816 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[32724, 32850, 32760, 32768, 32411, 33096, 32767, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 32247, 33098, 33096, 32779, 32305, 32429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=34816 expected_recv=64466 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 32247, 33098, 33096, 32779, 32305, 32429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=34816 expected_recv=65535 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=34816 expected_recv=65545 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=34816 expected_recv=64466 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65024 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 32247, 33098, 33096, 32779, 32305, 32429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=34816 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 32247, 33098, 33096, 32779, 32305, 32429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 32247, 33098, 33096, 32779, 32305, 32429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=34816 expected_recv=65543 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=34816 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32801, 32430, 33096, 33096, 32412, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=34816 expected_recv=64770 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 32247, 33098, 33096, 32779, 32305, 32429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=34816 expected_recv=64856 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32801, 32430, 33096, 33096, 32412, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=34816 expected_recv=64741 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 32247, 33098, 33096, 32779, 32305, 32429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=34816 expected_recv=65604 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32801, 32430, 33096, 33096, 32412, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=34816 expected_recv=65535 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=34816 expected_recv=64770 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=34816 expected_recv=64856 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=34816 expected_recv=64741 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=34816 expected_recv=65604 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32801, 32430, 33096, 33096, 32412, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32801, 32430, 33096, 33096, 32412, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32801, 32430, 33096, 33096, 32412, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32776, 32392, 32778, 32826, 32422] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=34816 expected_recv=64865 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32801, 32430, 33096, 33096, 32412, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=34816 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32776, 32392, 32778, 32826, 32422] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=34816 expected_recv=64852 expected_overflow=0 exact_recv_per_rank=[32777, 33096, 32801, 32430, 33096, 33096, 32412, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32776, 32392, 32778, 32826, 32422] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=34816 expected_recv=64865 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=34816 expected_recv=65543 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=34816 expected_recv=64852 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=34816 expected_recv=64885 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32776, 32392, 32778, 32826, 32422] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=34816 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32776, 32392, 32778, 32826, 32422] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=34816 expected_recv=64885 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=34816 expected_recv=65546 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=34816 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32776, 32392, 32778, 32826, 32422] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=34816 expected_recv=65527 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=34816 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[32774, 32773, 32755, 32724, 32452, 33094, 33096, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=34816 expected_recv=65452 expected_overflow=0 exact_recv_per_rank=[32774, 32773, 32755, 32724, 32452, 33094, 33096, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=34816 expected_recv=64809 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32776, 32392, 32778, 32826, 32422] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=34816 expected_recv=65542 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=34816 expected_recv=65594 expected_overflow=0 exact_recv_per_rank=[33095, 32759, 33096, 32776, 32392, 32778, 32826, 32422] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=34816 expected_recv=65514 expected_overflow=0 exact_recv_per_rank=[32774, 32773, 32755, 32724, 32452, 33094, 33096, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=34816 expected_recv=65452 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=34816 expected_recv=64809 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=34816 expected_recv=64904 expected_overflow=0 exact_recv_per_rank=[32774, 32773, 32755, 32724, 32452, 33094, 33096, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=34816 expected_recv=65594 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=34816 expected_recv=65514 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=34816 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[32774, 32773, 32755, 32724, 32452, 33094, 33096, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=34816 expected_recv=64904 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=34816 expected_recv=66190 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=34816 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[32774, 32773, 32755, 32724, 32452, 33094, 33096, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=34816 expected_recv=65540 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32438, 32451, 32772, 33096, 32766, 33096, 32429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=34816 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[33096, 32438, 32451, 32772, 33096, 32766, 33096, 32429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=34816 expected_recv=64954 expected_overflow=0 exact_recv_per_rank=[32774, 32773, 32755, 32724, 32452, 33094, 33096, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=34816 expected_recv=64891 expected_overflow=0 exact_recv_per_rank=[33096, 32438, 32451, 32772, 33096, 32766, 33096, 32429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=34816 expected_recv=65540 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32774, 32773, 32755, 32724, 32452, 33094, 33096, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32438, 32451, 32772, 33096, 32766, 33096, 32429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=34816 expected_recv=64954 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=34816 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[33096, 32438, 32451, 32772, 33096, 32766, 33096, 32429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=34816 expected_recv=64891 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=34816 expected_recv=65534 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=34816 expected_recv=64878 expected_overflow=0 exact_recv_per_rank=[33096, 32438, 32451, 32772, 33096, 32766, 33096, 32429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=34816 expected_recv=64878 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=34816 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[33096, 32882, 33096, 33097, 32003, 32767, 32444, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 32882, 33096, 33097, 32003, 32767, 32444, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=34816 expected_recv=66190 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=34816 expected_recv=64869 expected_overflow=0 exact_recv_per_rank=[33096, 32438, 32451, 32772, 33096, 32766, 33096, 32429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32882, 33096, 33097, 32003, 32767, 32444, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=34816 expected_recv=64443 expected_overflow=0 exact_recv_per_rank=[33096, 32882, 33096, 33097, 32003, 32767, 32444, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=34816 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[33096, 32882, 33096, 33097, 32003, 32767, 32444, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32438, 32451, 32772, 33096, 32766, 33096, 32429] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=34816 expected_recv=64869 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=34816 expected_recv=64443 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=34816 expected_recv=65535 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=34816 expected_recv=65790 expected_overflow=0 exact_recv_per_rank=[33096, 32882, 33096, 33097, 32003, 32767, 32444, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=34816 expected_recv=65790 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=34816 expected_recv=64849 expected_overflow=0 exact_recv_per_rank=[32378, 33096, 32780, 32375, 33096, 33096, 32948, 32375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=34816 expected_recv=64849 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=34816 expected_recv=65142 expected_overflow=0 exact_recv_per_rank=[32378, 33096, 32780, 32375, 33096, 33096, 32948, 32375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=34816 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[33096, 32882, 33096, 33097, 32003, 32767, 32444, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=34816 expected_recv=65142 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=34816 expected_recv=65548 expected_overflow=0 exact_recv_per_rank=[32378, 33096, 32780, 32375, 33096, 33096, 32948, 32375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32378, 33096, 32780, 32375, 33096, 33096, 32948, 32375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=34816 expected_recv=65864 expected_overflow=0 exact_recv_per_rank=[32378, 33096, 32780, 32375, 33096, 33096, 32948, 32375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=34816 expected_recv=64420 expected_overflow=0 exact_recv_per_rank=[33096, 32882, 33096, 33097, 32003, 32767, 32444, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=34816 expected_recv=65527 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=34816 expected_recv=65548 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=34816 expected_recv=65864 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=34816 expected_recv=64420 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32378, 33096, 32780, 32375, 33096, 33096, 32948, 32375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=64896 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=34816 expected_recv=64907 expected_overflow=0 exact_recv_per_rank=[32467, 32431, 33096, 32417, 33096, 32782, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=34816 expected_recv=64907 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=34816 expected_recv=64857 expected_overflow=0 exact_recv_per_rank=[32467, 32431, 33096, 32417, 33096, 32782, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=34816 expected_recv=64785 expected_overflow=0 exact_recv_per_rank=[32378, 33096, 32780, 32375, 33096, 33096, 32948, 32375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=34816 expected_recv=64857 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32467, 32431, 33096, 32417, 33096, 32782, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=34816 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[32467, 32431, 33096, 32417, 33096, 32782, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32467, 32431, 33096, 32417, 33096, 32782, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=34816 expected_recv=65716 expected_overflow=0 exact_recv_per_rank=[32378, 33096, 32780, 32375, 33096, 33096, 32948, 32375] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=34816 expected_recv=64785 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=34816 expected_recv=65550 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=34816 expected_recv=65716 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=34816 expected_recv=64871 expected_overflow=0 exact_recv_per_rank=[32467, 32431, 33096, 32417, 33096, 32782, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=34816 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[32781, 32755, 32782, 32781, 32461, 33096, 33096, 32392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=34816 expected_recv=64871 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=34816 expected_recv=65550 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=34816 expected_recv=65548 expected_overflow=0 exact_recv_per_rank=[32781, 32755, 32782, 32781, 32461, 33096, 33096, 32392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=34816 expected_recv=65548 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32467, 32431, 33096, 32417, 33096, 32782, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=34816 expected_recv=65547 expected_overflow=0 exact_recv_per_rank=[32781, 32755, 32782, 32781, 32461, 33096, 33096, 32392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=34816 expected_recv=64886 expected_overflow=0 exact_recv_per_rank=[32781, 32755, 32782, 32781, 32461, 33096, 33096, 32392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32781, 32755, 32782, 32781, 32461, 33096, 33096, 32392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=34816 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[32467, 32431, 33096, 32417, 33096, 32782, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=34816 expected_recv=65547 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=34816 expected_recv=64886 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=34816 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[32781, 32755, 32782, 32781, 32461, 33096, 33096, 32392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=34816 expected_recv=65527 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32841, 32917, 32755, 32767, 31905, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=34816 expected_recv=65523 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=34816 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[33096, 32841, 32917, 32755, 32767, 31905, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=34816 expected_recv=65523 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=34816 expected_recv=64850 expected_overflow=0 exact_recv_per_rank=[32781, 32755, 32782, 32781, 32461, 33096, 33096, 32392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=34816 expected_recv=65955 expected_overflow=0 exact_recv_per_rank=[33096, 32841, 32917, 32755, 32767, 31905, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=34816 expected_recv=65205 expected_overflow=0 exact_recv_per_rank=[33096, 32841, 32917, 32755, 32767, 31905, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=34816 expected_recv=64850 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=34816 expected_recv=64083 expected_overflow=0 exact_recv_per_rank=[33096, 32841, 32917, 32755, 32767, 31905, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=34816 expected_recv=65955 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32781, 32755, 32782, 32781, 32461, 33096, 33096, 32392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=34816 expected_recv=65205 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=34816 expected_recv=64083 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=34816 expected_recv=65603 expected_overflow=0 exact_recv_per_rank=[33096, 32841, 32917, 32755, 32767, 31905, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=64768 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=34816 expected_recv=65603 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[22087, 30221, 15319, 38011, 28899, 9128, 75640, 42839] avg=32768.0000 max=75640 max_rank=6 min=9128 min_rank=5 imbalance_ratio=2.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[22087, 30221, 15319, 38011, 28899, 9128, 75640, 42839] avg=32768.0000 max=75640 max_rank=6 min=9128 min_rank=5 imbalance_ratio=2.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32841, 32917, 32755, 32767, 31905, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[32275, 31293, 33097, 33096, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31293 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=42543 max_load_reduction_pct=0.562441 moved_to_replicas=113168 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[22087, 30221, 15319, 38011, 28899, 9128, 75640, 42839] avg=32768.0000 max=75640 max_rank=6 min=9128 min_rank=5 imbalance_ratio=2.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[22087, 30221, 15319, 38011, 28899, 9128, 75640, 42839] avg=32768.0000 max=75640 max_rank=6 min=9128 min_rank=5 imbalance_ratio=2.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[22087, 30221, 15319, 38011, 28899, 9128, 75640, 42839] avg=32768.0000 max=75640 max_rank=6 min=9128 min_rank=5 imbalance_ratio=2.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=34816 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[33096, 32841, 32917, 32755, 32767, 31905, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[32275, 31293, 33097, 33096, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31293 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=42543 max_load_reduction_pct=0.562441 moved_to_replicas=113168 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[22087, 30221, 15319, 38011, 28899, 9128, 75640, 42839] avg=32768.0000 max=75640 max_rank=6 min=9128 min_rank=5 imbalance_ratio=2.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=34816 expected_recv=65535 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[32275, 31293, 33097, 33096, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31293 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=42543 max_load_reduction_pct=0.562441 moved_to_replicas=113168 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[32275, 31293, 33097, 33096, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31293 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=42543 max_load_reduction_pct=0.562441 moved_to_replicas=113168 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[32275, 31293, 33097, 33096, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31293 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=42543 max_load_reduction_pct=0.562441 moved_to_replicas=113168 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[97690, 32764, 32753, 32810, 32759, 20, 94, 33254] avg=32768.0000 max=97690 max_rank=0 min=20 min_rank=5 imbalance_ratio=2.981262 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[32275, 31293, 33097, 33096, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31293 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=42543 max_load_reduction_pct=0.562441 moved_to_replicas=113168 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[97690, 32764, 32753, 32810, 32759, 20, 94, 33254] avg=32768.0000 max=97690 max_rank=0 min=20 min_rank=5 imbalance_ratio=2.981262 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[22087, 30221, 15319, 38011, 28899, 9128, 75640, 42839] avg=32768.0000 max=75640 max_rank=6 min=9128 min_rank=5 imbalance_ratio=2.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[33096, 32764, 32753, 32810, 32759, 32771, 32095, 33096] avg=32768.0000 max=33096 max_rank=0 min=32095 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64594 max_load_reduction_pct=0.661214 moved_to_replicas=129599 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[33096, 32764, 32753, 32810, 32759, 32771, 32095, 33096] avg=32768.0000 max=33096 max_rank=0 min=32095 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64594 max_load_reduction_pct=0.661214 moved_to_replicas=129599 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[97690, 32764, 32753, 32810, 32759, 20, 94, 33254] avg=32768.0000 max=97690 max_rank=0 min=20 min_rank=5 imbalance_ratio=2.981262 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[97690, 32764, 32753, 32810, 32759, 20, 94, 33254] avg=32768.0000 max=97690 max_rank=0 min=20 min_rank=5 imbalance_ratio=2.981262 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[97690, 32764, 32753, 32810, 32759, 20, 94, 33254] avg=32768.0000 max=97690 max_rank=0 min=20 min_rank=5 imbalance_ratio=2.981262 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[22087, 30221, 15319, 38011, 28899, 9128, 75640, 42839] avg=32768.0000 max=75640 max_rank=6 min=9128 min_rank=5 imbalance_ratio=2.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[97690, 32764, 32753, 32810, 32759, 20, 94, 33254] avg=32768.0000 max=97690 max_rank=0 min=20 min_rank=5 imbalance_ratio=2.981262 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[32275, 31293, 33097, 33096, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31293 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=42543 max_load_reduction_pct=0.562441 moved_to_replicas=113168 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[33096, 32764, 32753, 32810, 32759, 32771, 32095, 33096] avg=32768.0000 max=33096 max_rank=0 min=32095 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64594 max_load_reduction_pct=0.661214 moved_to_replicas=129599 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[33096, 32764, 32753, 32810, 32759, 32771, 32095, 33096] avg=32768.0000 max=33096 max_rank=0 min=32095 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64594 max_load_reduction_pct=0.661214 moved_to_replicas=129599 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[33096, 32764, 32753, 32810, 32759, 32771, 32095, 33096] avg=32768.0000 max=33096 max_rank=0 min=32095 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64594 max_load_reduction_pct=0.661214 moved_to_replicas=129599 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[65486, 32774, 98184, 32771, 14, 35, 32738, 142] avg=32768.0000 max=98184 max_rank=2 min=14 min_rank=4 imbalance_ratio=2.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[32275, 31293, 33097, 33096, 33096, 33095, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31293 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=42543 max_load_reduction_pct=0.562441 moved_to_replicas=113168 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[33096, 32764, 32753, 32810, 32759, 32771, 32095, 33096] avg=32768.0000 max=33096 max_rank=0 min=32095 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64594 max_load_reduction_pct=0.661214 moved_to_replicas=129599 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[65486, 32774, 98184, 32771, 14, 35, 32738, 142] avg=32768.0000 max=98184 max_rank=2 min=14 min_rank=4 imbalance_ratio=2.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[97690, 32764, 32753, 32810, 32759, 20, 94, 33254] avg=32768.0000 max=97690 max_rank=0 min=20 min_rank=5 imbalance_ratio=2.981262 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32774, 33096, 32771, 32772, 32365, 32738, 32532] avg=32768.0000 max=33096 max_rank=0 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65088 max_load_reduction_pct=0.662919 moved_to_replicas=195015 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[65486, 32774, 98184, 32771, 14, 35, 32738, 142] avg=32768.0000 max=98184 max_rank=2 min=14 min_rank=4 imbalance_ratio=2.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32774, 33096, 32771, 32772, 32365, 32738, 32532] avg=32768.0000 max=33096 max_rank=0 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65088 max_load_reduction_pct=0.662919 moved_to_replicas=195015 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[65486, 32774, 98184, 32771, 14, 35, 32738, 142] avg=32768.0000 max=98184 max_rank=2 min=14 min_rank=4 imbalance_ratio=2.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[65486, 32774, 98184, 32771, 14, 35, 32738, 142] avg=32768.0000 max=98184 max_rank=2 min=14 min_rank=4 imbalance_ratio=2.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[65486, 32774, 98184, 32771, 14, 35, 32738, 142] avg=32768.0000 max=98184 max_rank=2 min=14 min_rank=4 imbalance_ratio=2.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[97690, 32764, 32753, 32810, 32759, 20, 94, 33254] avg=32768.0000 max=97690 max_rank=0 min=20 min_rank=5 imbalance_ratio=2.981262 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[33096, 32764, 32753, 32810, 32759, 32771, 32095, 33096] avg=32768.0000 max=33096 max_rank=0 min=32095 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64594 max_load_reduction_pct=0.661214 moved_to_replicas=129599 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32774, 33096, 32771, 32772, 32365, 32738, 32532] avg=32768.0000 max=33096 max_rank=0 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65088 max_load_reduction_pct=0.662919 moved_to_replicas=195015 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32774, 33096, 32771, 32772, 32365, 32738, 32532] avg=32768.0000 max=33096 max_rank=0 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65088 max_load_reduction_pct=0.662919 moved_to_replicas=195015 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32774, 33096, 32771, 32772, 32365, 32738, 32532] avg=32768.0000 max=33096 max_rank=0 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65088 max_load_reduction_pct=0.662919 moved_to_replicas=195015 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[32673, 13655, 46789, 8, 65519, 32841, 68295, 2364] avg=32768.0000 max=68295 max_rank=6 min=8 min_rank=3 imbalance_ratio=2.084198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32774, 33096, 32771, 32772, 32365, 32738, 32532] avg=32768.0000 max=33096 max_rank=0 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65088 max_load_reduction_pct=0.662919 moved_to_replicas=195015 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[32673, 13655, 46789, 8, 65519, 32841, 68295, 2364] avg=32768.0000 max=68295 max_rank=6 min=8 min_rank=3 imbalance_ratio=2.084198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[33096, 32764, 32753, 32810, 32759, 32771, 32095, 33096] avg=32768.0000 max=33096 max_rank=0 min=32095 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64594 max_load_reduction_pct=0.661214 moved_to_replicas=129599 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[65486, 32774, 98184, 32771, 14, 35, 32738, 142] avg=32768.0000 max=98184 max_rank=2 min=14 min_rank=4 imbalance_ratio=2.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[32673, 31479, 33096, 32769, 33096, 32841, 33096, 33094] avg=32768.0000 max=33096 max_rank=2 min=31479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35199 max_load_reduction_pct=0.515396 moved_to_replicas=161436 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[32673, 13655, 46789, 8, 65519, 32841, 68295, 2364] avg=32768.0000 max=68295 max_rank=6 min=8 min_rank=3 imbalance_ratio=2.084198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[32673, 13655, 46789, 8, 65519, 32841, 68295, 2364] avg=32768.0000 max=68295 max_rank=6 min=8 min_rank=3 imbalance_ratio=2.084198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[32673, 31479, 33096, 32769, 33096, 32841, 33096, 33094] avg=32768.0000 max=33096 max_rank=2 min=31479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35199 max_load_reduction_pct=0.515396 moved_to_replicas=161436 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[32673, 13655, 46789, 8, 65519, 32841, 68295, 2364] avg=32768.0000 max=68295 max_rank=6 min=8 min_rank=3 imbalance_ratio=2.084198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[32673, 13655, 46789, 8, 65519, 32841, 68295, 2364] avg=32768.0000 max=68295 max_rank=6 min=8 min_rank=3 imbalance_ratio=2.084198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32774, 33096, 32771, 32772, 32365, 32738, 32532] avg=32768.0000 max=33096 max_rank=0 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65088 max_load_reduction_pct=0.662919 moved_to_replicas=195015 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[32673, 31479, 33096, 32769, 33096, 32841, 33096, 33094] avg=32768.0000 max=33096 max_rank=2 min=31479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35199 max_load_reduction_pct=0.515396 moved_to_replicas=161436 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[65486, 32774, 98184, 32771, 14, 35, 32738, 142] avg=32768.0000 max=98184 max_rank=2 min=14 min_rank=4 imbalance_ratio=2.996338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[32673, 31479, 33096, 32769, 33096, 32841, 33096, 33094] avg=32768.0000 max=33096 max_rank=2 min=31479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35199 max_load_reduction_pct=0.515396 moved_to_replicas=161436 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[32673, 31479, 33096, 32769, 33096, 32841, 33096, 33094] avg=32768.0000 max=33096 max_rank=2 min=31479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35199 max_load_reduction_pct=0.515396 moved_to_replicas=161436 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[20, 32772, 33154, 32779, 32738, 32770, 32796, 65115] avg=32768.0000 max=65115 max_rank=7 min=20 min_rank=0 imbalance_ratio=1.987152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[32673, 31479, 33096, 32769, 33096, 32841, 33096, 33094] avg=32768.0000 max=33096 max_rank=2 min=31479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35199 max_load_reduction_pct=0.515396 moved_to_replicas=161436 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[20, 32772, 33154, 32779, 32738, 32770, 32796, 65115] avg=32768.0000 max=65115 max_rank=7 min=20 min_rank=0 imbalance_ratio=1.987152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32774, 33096, 32771, 32772, 32365, 32738, 32532] avg=32768.0000 max=33096 max_rank=0 min=32365 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65088 max_load_reduction_pct=0.662919 moved_to_replicas=195015 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[32097, 32772, 33096, 32779, 32738, 32770, 32796, 33096] avg=32768.0000 max=33096 max_rank=2 min=32097 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32019 max_load_reduction_pct=0.491730 moved_to_replicas=64287 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[20, 32772, 33154, 32779, 32738, 32770, 32796, 65115] avg=32768.0000 max=65115 max_rank=7 min=20 min_rank=0 imbalance_ratio=1.987152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[32673, 13655, 46789, 8, 65519, 32841, 68295, 2364] avg=32768.0000 max=68295 max_rank=6 min=8 min_rank=3 imbalance_ratio=2.084198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[20, 32772, 33154, 32779, 32738, 32770, 32796, 65115] avg=32768.0000 max=65115 max_rank=7 min=20 min_rank=0 imbalance_ratio=1.987152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[32097, 32772, 33096, 32779, 32738, 32770, 32796, 33096] avg=32768.0000 max=33096 max_rank=2 min=32097 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32019 max_load_reduction_pct=0.491730 moved_to_replicas=64287 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[20, 32772, 33154, 32779, 32738, 32770, 32796, 65115] avg=32768.0000 max=65115 max_rank=7 min=20 min_rank=0 imbalance_ratio=1.987152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[20, 32772, 33154, 32779, 32738, 32770, 32796, 65115] avg=32768.0000 max=65115 max_rank=7 min=20 min_rank=0 imbalance_ratio=1.987152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[32097, 32772, 33096, 32779, 32738, 32770, 32796, 33096] avg=32768.0000 max=33096 max_rank=2 min=32097 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32019 max_load_reduction_pct=0.491730 moved_to_replicas=64287 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[32097, 32772, 33096, 32779, 32738, 32770, 32796, 33096] avg=32768.0000 max=33096 max_rank=2 min=32097 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32019 max_load_reduction_pct=0.491730 moved_to_replicas=64287 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[32673, 31479, 33096, 32769, 33096, 32841, 33096, 33094] avg=32768.0000 max=33096 max_rank=2 min=31479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35199 max_load_reduction_pct=0.515396 moved_to_replicas=161436 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[32673, 13655, 46789, 8, 65519, 32841, 68295, 2364] avg=32768.0000 max=68295 max_rank=6 min=8 min_rank=3 imbalance_ratio=2.084198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[32097, 32772, 33096, 32779, 32738, 32770, 32796, 33096] avg=32768.0000 max=33096 max_rank=2 min=32097 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32019 max_load_reduction_pct=0.491730 moved_to_replicas=64287 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[32724, 32850, 32760, 32768, 20, 98247, 32767, 8] avg=32768.0000 max=98247 max_rank=5 min=8 min_rank=7 imbalance_ratio=2.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[32097, 32772, 33096, 32779, 32738, 32770, 32796, 33096] avg=32768.0000 max=33096 max_rank=2 min=32097 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32019 max_load_reduction_pct=0.491730 moved_to_replicas=64287 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[32724, 32850, 32760, 32768, 20, 98247, 32767, 8] avg=32768.0000 max=98247 max_rank=5 min=8 min_rank=7 imbalance_ratio=2.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[32673, 31479, 33096, 32769, 33096, 32841, 33096, 33094] avg=32768.0000 max=33096 max_rank=2 min=31479 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=35199 max_load_reduction_pct=0.515396 moved_to_replicas=161436 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[32724, 32850, 32760, 32768, 20, 98247, 32767, 8] avg=32768.0000 max=98247 max_rank=5 min=8 min_rank=7 imbalance_ratio=2.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[32724, 32850, 32760, 32768, 20, 98247, 32767, 8] avg=32768.0000 max=98247 max_rank=5 min=8 min_rank=7 imbalance_ratio=2.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[20, 32772, 33154, 32779, 32738, 32770, 32796, 65115] avg=32768.0000 max=65115 max_rank=7 min=20 min_rank=0 imbalance_ratio=1.987152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[32724, 32850, 32760, 32768, 32411, 33096, 32767, 32768] avg=32768.0000 max=33096 max_rank=5 min=32411 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65151 max_load_reduction_pct=0.663135 moved_to_replicas=130349 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[32724, 32850, 32760, 32768, 32411, 33096, 32767, 32768] avg=32768.0000 max=33096 max_rank=5 min=32411 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65151 max_load_reduction_pct=0.663135 moved_to_replicas=130349 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[32724, 32850, 32760, 32768, 20, 98247, 32767, 8] avg=32768.0000 max=98247 max_rank=5 min=8 min_rank=7 imbalance_ratio=2.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[32724, 32850, 32760, 32768, 20, 98247, 32767, 8] avg=32768.0000 max=98247 max_rank=5 min=8 min_rank=7 imbalance_ratio=2.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[32724, 32850, 32760, 32768, 32411, 33096, 32767, 32768] avg=32768.0000 max=33096 max_rank=5 min=32411 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65151 max_load_reduction_pct=0.663135 moved_to_replicas=130349 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[32724, 32850, 32760, 32768, 32411, 33096, 32767, 32768] avg=32768.0000 max=33096 max_rank=5 min=32411 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65151 max_load_reduction_pct=0.663135 moved_to_replicas=130349 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[32097, 32772, 33096, 32779, 32738, 32770, 32796, 33096] avg=32768.0000 max=33096 max_rank=2 min=32097 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32019 max_load_reduction_pct=0.491730 moved_to_replicas=64287 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[20, 32772, 33154, 32779, 32738, 32770, 32796, 65115] avg=32768.0000 max=65115 max_rank=7 min=20 min_rank=0 imbalance_ratio=1.987152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[32724, 32850, 32760, 32768, 32411, 33096, 32767, 32768] avg=32768.0000 max=33096 max_rank=5 min=32411 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65151 max_load_reduction_pct=0.663135 moved_to_replicas=130349 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[65515, 27090, 30584, 40765, 65385, 32779, 16, 10] avg=32768.0000 max=65515 max_rank=0 min=10 min_rank=7 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[32724, 32850, 32760, 32768, 32411, 33096, 32767, 32768] avg=32768.0000 max=33096 max_rank=5 min=32411 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65151 max_load_reduction_pct=0.663135 moved_to_replicas=130349 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[65515, 27090, 30584, 40765, 65385, 32779, 16, 10] avg=32768.0000 max=65515 max_rank=0 min=10 min_rank=7 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[32097, 32772, 33096, 32779, 32738, 32770, 32796, 33096] avg=32768.0000 max=33096 max_rank=2 min=32097 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32019 max_load_reduction_pct=0.491730 moved_to_replicas=64287 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[65515, 27090, 30584, 40765, 65385, 32779, 16, 10] avg=32768.0000 max=65515 max_rank=0 min=10 min_rank=7 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[65515, 27090, 30584, 40765, 65385, 32779, 16, 10] avg=32768.0000 max=65515 max_rank=0 min=10 min_rank=7 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[32724, 32850, 32760, 32768, 20, 98247, 32767, 8] avg=32768.0000 max=98247 max_rank=5 min=8 min_rank=7 imbalance_ratio=2.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[33096, 33094, 32247, 33098, 33096, 32779, 32305, 32429] avg=32768.0000 max=33098 max_rank=3 min=32247 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32417 max_load_reduction_pct=0.494803 moved_to_replicas=143728 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[33096, 33094, 32247, 33098, 33096, 32779, 32305, 32429] avg=32768.0000 max=33098 max_rank=3 min=32247 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32417 max_load_reduction_pct=0.494803 moved_to_replicas=143728 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[65515, 27090, 30584, 40765, 65385, 32779, 16, 10] avg=32768.0000 max=65515 max_rank=0 min=10 min_rank=7 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[65515, 27090, 30584, 40765, 65385, 32779, 16, 10] avg=32768.0000 max=65515 max_rank=0 min=10 min_rank=7 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[33096, 33094, 32247, 33098, 33096, 32779, 32305, 32429] avg=32768.0000 max=33098 max_rank=3 min=32247 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32417 max_load_reduction_pct=0.494803 moved_to_replicas=143728 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[33096, 33094, 32247, 33098, 33096, 32779, 32305, 32429] avg=32768.0000 max=33098 max_rank=3 min=32247 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32417 max_load_reduction_pct=0.494803 moved_to_replicas=143728 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[32724, 32850, 32760, 32768, 32411, 33096, 32767, 32768] avg=32768.0000 max=33096 max_rank=5 min=32411 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65151 max_load_reduction_pct=0.663135 moved_to_replicas=130349 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[32724, 32850, 32760, 32768, 20, 98247, 32767, 8] avg=32768.0000 max=98247 max_rank=5 min=8 min_rank=7 imbalance_ratio=2.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[32777, 65494, 32801, 888, 65522, 64638, 14, 10] avg=32768.0000 max=65522 max_rank=4 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[33096, 33094, 32247, 33098, 33096, 32779, 32305, 32429] avg=32768.0000 max=33098 max_rank=3 min=32247 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32417 max_load_reduction_pct=0.494803 moved_to_replicas=143728 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[32777, 65494, 32801, 888, 65522, 64638, 14, 10] avg=32768.0000 max=65522 max_rank=4 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[33096, 33094, 32247, 33098, 33096, 32779, 32305, 32429] avg=32768.0000 max=33098 max_rank=3 min=32247 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32417 max_load_reduction_pct=0.494803 moved_to_replicas=143728 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[32724, 32850, 32760, 32768, 32411, 33096, 32767, 32768] avg=32768.0000 max=33096 max_rank=5 min=32411 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65151 max_load_reduction_pct=0.663135 moved_to_replicas=130349 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[32777, 65494, 32801, 888, 65522, 64638, 14, 10] avg=32768.0000 max=65522 max_rank=4 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[32777, 65494, 32801, 888, 65522, 64638, 14, 10] avg=32768.0000 max=65522 max_rank=4 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[65515, 27090, 30584, 40765, 65385, 32779, 16, 10] avg=32768.0000 max=65515 max_rank=0 min=10 min_rank=7 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[32777, 33096, 32801, 32430, 33096, 33096, 32412, 32436] avg=32768.0000 max=33096 max_rank=1 min=32412 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=191014 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[32777, 33096, 32801, 32430, 33096, 33096, 32412, 32436] avg=32768.0000 max=33096 max_rank=1 min=32412 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=191014 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[32777, 65494, 32801, 888, 65522, 64638, 14, 10] avg=32768.0000 max=65522 max_rank=4 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[32777, 33096, 32801, 32430, 33096, 33096, 32412, 32436] avg=32768.0000 max=33096 max_rank=1 min=32412 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=191014 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[32777, 65494, 32801, 888, 65522, 64638, 14, 10] avg=32768.0000 max=65522 max_rank=4 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[32777, 33096, 32801, 32430, 33096, 33096, 32412, 32436] avg=32768.0000 max=33096 max_rank=1 min=32412 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=191014 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[33096, 33094, 32247, 33098, 33096, 32779, 32305, 32429] avg=32768.0000 max=33098 max_rank=3 min=32247 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32417 max_load_reduction_pct=0.494803 moved_to_replicas=143728 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[65515, 27090, 30584, 40765, 65385, 32779, 16, 10] avg=32768.0000 max=65515 max_rank=0 min=10 min_rank=7 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[32777, 33096, 32801, 32430, 33096, 33096, 32412, 32436] avg=32768.0000 max=33096 max_rank=1 min=32412 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=191014 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[8512, 32759, 98271, 17, 32392, 32778, 32826, 24589] avg=32768.0000 max=98271 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[8512, 32759, 98271, 17, 32392, 32778, 32826, 24589] avg=32768.0000 max=98271 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[32777, 33096, 32801, 32430, 33096, 33096, 32412, 32436] avg=32768.0000 max=33096 max_rank=1 min=32412 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=191014 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[33096, 33094, 32247, 33098, 33096, 32779, 32305, 32429] avg=32768.0000 max=33098 max_rank=3 min=32247 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=32417 max_load_reduction_pct=0.494803 moved_to_replicas=143728 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[8512, 32759, 98271, 17, 32392, 32778, 32826, 24589] avg=32768.0000 max=98271 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[8512, 32759, 98271, 17, 32392, 32778, 32826, 24589] avg=32768.0000 max=98271 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[32777, 65494, 32801, 888, 65522, 64638, 14, 10] avg=32768.0000 max=65522 max_rank=4 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[33095, 32759, 33096, 32776, 32392, 32778, 32826, 32422] avg=32768.0000 max=33096 max_rank=2 min=32392 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65175 max_load_reduction_pct=0.663217 moved_to_replicas=130372 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[33095, 32759, 33096, 32776, 32392, 32778, 32826, 32422] avg=32768.0000 max=33096 max_rank=2 min=32392 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65175 max_load_reduction_pct=0.663217 moved_to_replicas=130372 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[8512, 32759, 98271, 17, 32392, 32778, 32826, 24589] avg=32768.0000 max=98271 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[33095, 32759, 33096, 32776, 32392, 32778, 32826, 32422] avg=32768.0000 max=33096 max_rank=2 min=32392 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65175 max_load_reduction_pct=0.663217 moved_to_replicas=130372 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[33095, 32759, 33096, 32776, 32392, 32778, 32826, 32422] avg=32768.0000 max=33096 max_rank=2 min=32392 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65175 max_load_reduction_pct=0.663217 moved_to_replicas=130372 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[8512, 32759, 98271, 17, 32392, 32778, 32826, 24589] avg=32768.0000 max=98271 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[32777, 33096, 32801, 32430, 33096, 33096, 32412, 32436] avg=32768.0000 max=33096 max_rank=1 min=32412 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=191014 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[32777, 65494, 32801, 888, 65522, 64638, 14, 10] avg=32768.0000 max=65522 max_rank=4 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[33095, 32759, 33096, 32776, 32392, 32778, 32826, 32422] avg=32768.0000 max=33096 max_rank=2 min=32392 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65175 max_load_reduction_pct=0.663217 moved_to_replicas=130372 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32773, 32755, 32724, 37, 44804, 65511, 20766] avg=32768.0000 max=65511 max_rank=6 min=37 min_rank=4 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32773, 32755, 32724, 37, 44804, 65511, 20766] avg=32768.0000 max=65511 max_rank=6 min=37 min_rank=4 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[33095, 32759, 33096, 32776, 32392, 32778, 32826, 32422] avg=32768.0000 max=33096 max_rank=2 min=32392 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65175 max_load_reduction_pct=0.663217 moved_to_replicas=130372 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32773, 32755, 32724, 37, 44804, 65511, 20766] avg=32768.0000 max=65511 max_rank=6 min=37 min_rank=4 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32773, 32755, 32724, 37, 44804, 65511, 20766] avg=32768.0000 max=65511 max_rank=6 min=37 min_rank=4 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[32777, 33096, 32801, 32430, 33096, 33096, 32412, 32436] avg=32768.0000 max=33096 max_rank=1 min=32412 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=191014 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[8512, 32759, 98271, 17, 32392, 32778, 32826, 24589] avg=32768.0000 max=98271 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32773, 32755, 32724, 32452, 33094, 33096, 32476] avg=32768.0000 max=33096 max_rank=6 min=32452 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=86544 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32773, 32755, 32724, 32452, 33094, 33096, 32476] avg=32768.0000 max=33096 max_rank=6 min=32452 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=86544 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32773, 32755, 32724, 32452, 33094, 33096, 32476] avg=32768.0000 max=33096 max_rank=6 min=32452 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=86544 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32773, 32755, 32724, 37, 44804, 65511, 20766] avg=32768.0000 max=65511 max_rank=6 min=37 min_rank=4 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32773, 32755, 32724, 32452, 33094, 33096, 32476] avg=32768.0000 max=33096 max_rank=6 min=32452 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=86544 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32773, 32755, 32724, 37, 44804, 65511, 20766] avg=32768.0000 max=65511 max_rank=6 min=37 min_rank=4 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[33095, 32759, 33096, 32776, 32392, 32778, 32826, 32422] avg=32768.0000 max=33096 max_rank=2 min=32392 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65175 max_load_reduction_pct=0.663217 moved_to_replicas=130372 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[8512, 32759, 98271, 17, 32392, 32778, 32826, 24589] avg=32768.0000 max=98271 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.998993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[65501, 6, 22, 32772, 65528, 32766, 65525, 24] avg=32768.0000 max=65528 max_rank=4 min=6 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[65501, 6, 22, 32772, 65528, 32766, 65525, 24] avg=32768.0000 max=65528 max_rank=4 min=6 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32773, 32755, 32724, 32452, 33094, 33096, 32476] avg=32768.0000 max=33096 max_rank=6 min=32452 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=86544 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32773, 32755, 32724, 32452, 33094, 33096, 32476] avg=32768.0000 max=33096 max_rank=6 min=32452 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=86544 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[65501, 6, 22, 32772, 65528, 32766, 65525, 24] avg=32768.0000 max=65528 max_rank=4 min=6 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[65501, 6, 22, 32772, 65528, 32766, 65525, 24] avg=32768.0000 max=65528 max_rank=4 min=6 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[33095, 32759, 33096, 32776, 32392, 32778, 32826, 32422] avg=32768.0000 max=33096 max_rank=2 min=32392 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65175 max_load_reduction_pct=0.663217 moved_to_replicas=130372 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[33096, 32438, 32451, 32772, 33096, 32766, 33096, 32429] avg=32768.0000 max=33096 max_rank=0 min=32429 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194586 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32773, 32755, 32724, 37, 44804, 65511, 20766] avg=32768.0000 max=65511 max_rank=6 min=37 min_rank=4 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[33096, 32438, 32451, 32772, 33096, 32766, 33096, 32429] avg=32768.0000 max=33096 max_rank=0 min=32429 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194586 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[33096, 32438, 32451, 32772, 33096, 32766, 33096, 32429] avg=32768.0000 max=33096 max_rank=0 min=32429 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194586 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[33096, 32438, 32451, 32772, 33096, 32766, 33096, 32429] avg=32768.0000 max=33096 max_rank=0 min=32429 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194586 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[65501, 6, 22, 32772, 65528, 32766, 65525, 24] avg=32768.0000 max=65528 max_rank=4 min=6 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[65501, 6, 22, 32772, 65528, 32766, 65525, 24] avg=32768.0000 max=65528 max_rank=4 min=6 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32773, 32755, 32724, 37, 44804, 65511, 20766] avg=32768.0000 max=65511 max_rank=6 min=37 min_rank=4 imbalance_ratio=1.999237 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32773, 32755, 32724, 32452, 33094, 33096, 32476] avg=32768.0000 max=33096 max_rank=6 min=32452 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=86544 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[56582, 32882, 65521, 41549, 65, 32767, 19, 32759] avg=32768.0000 max=65521 max_rank=2 min=19 min_rank=6 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[33096, 32438, 32451, 32772, 33096, 32766, 33096, 32429] avg=32768.0000 max=33096 max_rank=0 min=32429 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194586 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[33096, 32438, 32451, 32772, 33096, 32766, 33096, 32429] avg=32768.0000 max=33096 max_rank=0 min=32429 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194586 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[56582, 32882, 65521, 41549, 65, 32767, 19, 32759] avg=32768.0000 max=65521 max_rank=2 min=19 min_rank=6 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[56582, 32882, 65521, 41549, 65, 32767, 19, 32759] avg=32768.0000 max=65521 max_rank=2 min=19 min_rank=6 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[56582, 32882, 65521, 41549, 65, 32767, 19, 32759] avg=32768.0000 max=65521 max_rank=2 min=19 min_rank=6 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32773, 32755, 32724, 32452, 33094, 33096, 32476] avg=32768.0000 max=33096 max_rank=6 min=32452 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32415 max_load_reduction_pct=0.494802 moved_to_replicas=86544 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[33096, 32882, 33096, 33097, 32003, 32767, 32444, 32759] avg=32768.0000 max=33097 max_rank=3 min=32003 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32424 max_load_reduction_pct=0.494864 moved_to_replicas=128776 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[33096, 32882, 33096, 33097, 32003, 32767, 32444, 32759] avg=32768.0000 max=33097 max_rank=3 min=32003 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32424 max_load_reduction_pct=0.494864 moved_to_replicas=128776 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[65501, 6, 22, 32772, 65528, 32766, 65525, 24] avg=32768.0000 max=65528 max_rank=4 min=6 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[33096, 32882, 33096, 33097, 32003, 32767, 32444, 32759] avg=32768.0000 max=33097 max_rank=3 min=32003 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32424 max_load_reduction_pct=0.494864 moved_to_replicas=128776 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[33096, 32882, 33096, 33097, 32003, 32767, 32444, 32759] avg=32768.0000 max=33097 max_rank=3 min=32003 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32424 max_load_reduction_pct=0.494864 moved_to_replicas=128776 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[56582, 32882, 65521, 41549, 65, 32767, 19, 32759] avg=32768.0000 max=65521 max_rank=2 min=19 min_rank=6 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[56582, 32882, 65521, 41549, 65, 32767, 19, 32759] avg=32768.0000 max=65521 max_rank=2 min=19 min_rank=6 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[33096, 32438, 32451, 32772, 33096, 32766, 33096, 32429] avg=32768.0000 max=33096 max_rank=0 min=32429 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194586 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[65501, 6, 22, 32772, 65528, 32766, 65525, 24] avg=32768.0000 max=65528 max_rank=4 min=6 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[52, 65377, 32780, 32375, 33107, 65411, 32948, 94] avg=32768.0000 max=65411 max_rank=5 min=52 min_rank=0 imbalance_ratio=1.996185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[33096, 32882, 33096, 33097, 32003, 32767, 32444, 32759] avg=32768.0000 max=33097 max_rank=3 min=32003 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32424 max_load_reduction_pct=0.494864 moved_to_replicas=128776 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[52, 65377, 32780, 32375, 33107, 65411, 32948, 94] avg=32768.0000 max=65411 max_rank=5 min=52 min_rank=0 imbalance_ratio=1.996185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[33096, 32882, 33096, 33097, 32003, 32767, 32444, 32759] avg=32768.0000 max=33097 max_rank=3 min=32003 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32424 max_load_reduction_pct=0.494864 moved_to_replicas=128776 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[52, 65377, 32780, 32375, 33107, 65411, 32948, 94] avg=32768.0000 max=65411 max_rank=5 min=52 min_rank=0 imbalance_ratio=1.996185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[52, 65377, 32780, 32375, 33107, 65411, 32948, 94] avg=32768.0000 max=65411 max_rank=5 min=52 min_rank=0 imbalance_ratio=1.996185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[33096, 32438, 32451, 32772, 33096, 32766, 33096, 32429] avg=32768.0000 max=33096 max_rank=0 min=32429 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194586 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[32378, 33096, 32780, 32375, 33096, 33096, 32948, 32375] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32315 max_load_reduction_pct=0.494030 moved_to_replicas=129483 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[32378, 33096, 32780, 32375, 33096, 33096, 32948, 32375] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32315 max_load_reduction_pct=0.494030 moved_to_replicas=129483 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[56582, 32882, 65521, 41549, 65, 32767, 19, 32759] avg=32768.0000 max=65521 max_rank=2 min=19 min_rank=6 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[32378, 33096, 32780, 32375, 33096, 33096, 32948, 32375] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32315 max_load_reduction_pct=0.494030 moved_to_replicas=129483 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[52, 65377, 32780, 32375, 33107, 65411, 32948, 94] avg=32768.0000 max=65411 max_rank=5 min=52 min_rank=0 imbalance_ratio=1.996185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[32378, 33096, 32780, 32375, 33096, 33096, 32948, 32375] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32315 max_load_reduction_pct=0.494030 moved_to_replicas=129483 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[52, 65377, 32780, 32375, 33107, 65411, 32948, 94] avg=32768.0000 max=65411 max_rank=5 min=52 min_rank=0 imbalance_ratio=1.996185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[33096, 32882, 33096, 33097, 32003, 32767, 32444, 32759] avg=32768.0000 max=33097 max_rank=3 min=32003 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32424 max_load_reduction_pct=0.494864 moved_to_replicas=128776 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[15, 85, 65548, 67, 65446, 32782, 32759, 65442] avg=32768.0000 max=65548 max_rank=2 min=15 min_rank=0 imbalance_ratio=2.000366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[56582, 32882, 65521, 41549, 65, 32767, 19, 32759] avg=32768.0000 max=65521 max_rank=2 min=19 min_rank=6 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[32378, 33096, 32780, 32375, 33096, 33096, 32948, 32375] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32315 max_load_reduction_pct=0.494030 moved_to_replicas=129483 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[15, 85, 65548, 67, 65446, 32782, 32759, 65442] avg=32768.0000 max=65548 max_rank=2 min=15 min_rank=0 imbalance_ratio=2.000366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[15, 85, 65548, 67, 65446, 32782, 32759, 65442] avg=32768.0000 max=65548 max_rank=2 min=15 min_rank=0 imbalance_ratio=2.000366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[32378, 33096, 32780, 32375, 33096, 33096, 32948, 32375] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32315 max_load_reduction_pct=0.494030 moved_to_replicas=129483 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[15, 85, 65548, 67, 65446, 32782, 32759, 65442] avg=32768.0000 max=65548 max_rank=2 min=15 min_rank=0 imbalance_ratio=2.000366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[32467, 32431, 33096, 32417, 33096, 32782, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=32417 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32452 max_load_reduction_pct=0.495088 moved_to_replicas=194450 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[33096, 32882, 33096, 33097, 32003, 32767, 32444, 32759] avg=32768.0000 max=33097 max_rank=3 min=32003 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32424 max_load_reduction_pct=0.494864 moved_to_replicas=128776 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[32467, 32431, 33096, 32417, 33096, 32782, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=32417 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32452 max_load_reduction_pct=0.495088 moved_to_replicas=194450 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[52, 65377, 32780, 32375, 33107, 65411, 32948, 94] avg=32768.0000 max=65411 max_rank=5 min=52 min_rank=0 imbalance_ratio=1.996185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[32467, 32431, 33096, 32417, 33096, 32782, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=32417 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32452 max_load_reduction_pct=0.495088 moved_to_replicas=194450 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[32467, 32431, 33096, 32417, 33096, 32782, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=32417 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32452 max_load_reduction_pct=0.495088 moved_to_replicas=194450 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[15, 85, 65548, 67, 65446, 32782, 32759, 65442] avg=32768.0000 max=65548 max_rank=2 min=15 min_rank=0 imbalance_ratio=2.000366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[15, 85, 65548, 67, 65446, 32782, 32759, 65442] avg=32768.0000 max=65548 max_rank=2 min=15 min_rank=0 imbalance_ratio=2.000366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[32781, 32755, 32782, 32781, 155, 65463, 65402, 25] avg=32768.0000 max=65463 max_rank=5 min=25 min_rank=7 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[32378, 33096, 32780, 32375, 33096, 33096, 32948, 32375] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32315 max_load_reduction_pct=0.494030 moved_to_replicas=129483 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[52, 65377, 32780, 32375, 33107, 65411, 32948, 94] avg=32768.0000 max=65411 max_rank=5 min=52 min_rank=0 imbalance_ratio=1.996185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[32781, 32755, 32782, 32781, 155, 65463, 65402, 25] avg=32768.0000 max=65463 max_rank=5 min=25 min_rank=7 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[32467, 32431, 33096, 32417, 33096, 32782, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=32417 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32452 max_load_reduction_pct=0.495088 moved_to_replicas=194450 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[32781, 32755, 32782, 32781, 155, 65463, 65402, 25] avg=32768.0000 max=65463 max_rank=5 min=25 min_rank=7 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[32467, 32431, 33096, 32417, 33096, 32782, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=32417 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32452 max_load_reduction_pct=0.495088 moved_to_replicas=194450 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[32781, 32755, 32782, 32781, 155, 65463, 65402, 25] avg=32768.0000 max=65463 max_rank=5 min=25 min_rank=7 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[32781, 32755, 32782, 32781, 32461, 33096, 33096, 32392] avg=32768.0000 max=33096 max_rank=5 min=32392 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=129503 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[32781, 32755, 32782, 32781, 32461, 33096, 33096, 32392] avg=32768.0000 max=33096 max_rank=5 min=32392 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=129503 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[32378, 33096, 32780, 32375, 33096, 33096, 32948, 32375] avg=32768.0000 max=33096 max_rank=1 min=32375 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32315 max_load_reduction_pct=0.494030 moved_to_replicas=129483 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[32781, 32755, 32782, 32781, 32461, 33096, 33096, 32392] avg=32768.0000 max=33096 max_rank=5 min=32392 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=129503 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[15, 85, 65548, 67, 65446, 32782, 32759, 65442] avg=32768.0000 max=65548 max_rank=2 min=15 min_rank=0 imbalance_ratio=2.000366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[32781, 32755, 32782, 32781, 32461, 33096, 33096, 32392] avg=32768.0000 max=33096 max_rank=5 min=32392 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=129503 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[32781, 32755, 32782, 32781, 155, 65463, 65402, 25] avg=32768.0000 max=65463 max_rank=5 min=25 min_rank=7 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[32781, 32755, 32782, 32781, 155, 65463, 65402, 25] avg=32768.0000 max=65463 max_rank=5 min=25 min_rank=7 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[67120, 32841, 32917, 32755, 23, 42, 32767, 63679] avg=32768.0000 max=67120 max_rank=0 min=23 min_rank=4 imbalance_ratio=2.048340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[67120, 32841, 32917, 32755, 23, 42, 32767, 63679] avg=32768.0000 max=67120 max_rank=0 min=23 min_rank=4 imbalance_ratio=2.048340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[32467, 32431, 33096, 32417, 33096, 32782, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=32417 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32452 max_load_reduction_pct=0.495088 moved_to_replicas=194450 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[15, 85, 65548, 67, 65446, 32782, 32759, 65442] avg=32768.0000 max=65548 max_rank=2 min=15 min_rank=0 imbalance_ratio=2.000366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[32781, 32755, 32782, 32781, 32461, 33096, 33096, 32392] avg=32768.0000 max=33096 max_rank=5 min=32392 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=129503 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[67120, 32841, 32917, 32755, 23, 42, 32767, 63679] avg=32768.0000 max=67120 max_rank=0 min=23 min_rank=4 imbalance_ratio=2.048340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[32781, 32755, 32782, 32781, 32461, 33096, 33096, 32392] avg=32768.0000 max=33096 max_rank=5 min=32392 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=129503 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[67120, 32841, 32917, 32755, 23, 42, 32767, 63679] avg=32768.0000 max=67120 max_rank=0 min=23 min_rank=4 imbalance_ratio=2.048340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32841, 32917, 32755, 32767, 31905, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31905 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34024 max_load_reduction_pct=0.506913 moved_to_replicas=129214 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32841, 32917, 32755, 32767, 31905, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31905 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34024 max_load_reduction_pct=0.506913 moved_to_replicas=129214 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[32467, 32431, 33096, 32417, 33096, 32782, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=32417 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32452 max_load_reduction_pct=0.495088 moved_to_replicas=194450 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32841, 32917, 32755, 32767, 31905, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31905 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34024 max_load_reduction_pct=0.506913 moved_to_replicas=129214 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[32781, 32755, 32782, 32781, 155, 65463, 65402, 25] avg=32768.0000 max=65463 max_rank=5 min=25 min_rank=7 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32841, 32917, 32755, 32767, 31905, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31905 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34024 max_load_reduction_pct=0.506913 moved_to_replicas=129214 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[67120, 32841, 32917, 32755, 23, 42, 32767, 63679] avg=32768.0000 max=67120 max_rank=0 min=23 min_rank=4 imbalance_ratio=2.048340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[67120, 32841, 32917, 32755, 23, 42, 32767, 63679] avg=32768.0000 max=67120 max_rank=0 min=23 min_rank=4 imbalance_ratio=2.048340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[32781, 32755, 32782, 32781, 32461, 33096, 33096, 32392] avg=32768.0000 max=33096 max_rank=5 min=32392 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=129503 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[32781, 32755, 32782, 32781, 155, 65463, 65402, 25] avg=32768.0000 max=65463 max_rank=5 min=25 min_rank=7 imbalance_ratio=1.997772 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32841, 32917, 32755, 32767, 31905, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31905 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34024 max_load_reduction_pct=0.506913 moved_to_replicas=129214 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32841, 32917, 32755, 32767, 31905, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31905 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34024 max_load_reduction_pct=0.506913 moved_to_replicas=129214 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[32781, 32755, 32782, 32781, 32461, 33096, 33096, 32392] avg=32768.0000 max=33096 max_rank=5 min=32392 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32367 max_load_reduction_pct=0.494432 moved_to_replicas=129503 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[67120, 32841, 32917, 32755, 23, 42, 32767, 63679] avg=32768.0000 max=67120 max_rank=0 min=23 min_rank=4 imbalance_ratio=2.048340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32841, 32917, 32755, 32767, 31905, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31905 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34024 max_load_reduction_pct=0.506913 moved_to_replicas=129214 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[67120, 32841, 32917, 32755, 23, 42, 32767, 63679] avg=32768.0000 max=67120 max_rank=0 min=23 min_rank=4 imbalance_ratio=2.048340 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32841, 32917, 32755, 32767, 31905, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31905 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34024 max_load_reduction_pct=0.506913 moved_to_replicas=129214 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=32 total_expected_recv=1050394 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56448 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=32 total_expected_recv=1050255 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=55552 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=32 total_expected_recv=1046132 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=60032 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=32 total_expected_recv=1048798 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=58880 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=32 total_expected_recv=1051802 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=54272 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=32 total_expected_recv=1045908 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=61056 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=32 total_expected_recv=1047177 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=58752 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=32 total_expected_recv=1048142 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=58624 peak_demand=33096 required_capacity=1.010010 + [2026-08-30 05:11:44] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 537.2 | throughput per GPU (TFLOP/s/GPU): 620.6 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.558580E+00 | loss scale: 1.0 | grad norm: 5.844 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=34816 expected_recv=65907 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33097, 33096, 33095, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33097, 33096, 33095, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33097, 33096, 33095, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33097, 33096, 33095, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=34816 expected_recv=61230 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33097, 33096, 33095, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33097, 33096, 33095, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33097, 33096, 33095, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=34816 expected_recv=65907 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33097, 33096, 33095, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=34816 expected_recv=61230 actual_valid_recv=61952 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=61952 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32731, 33096, 33096, 32758, 32740, 31531, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32731, 33096, 33096, 32758, 32740, 31531, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=34816 expected_recv=65499 expected_overflow=0 exact_recv_per_rank=[33096, 32731, 33096, 33096, 32758, 32740, 31531, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32731, 33096, 33096, 32758, 32740, 31531, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=34816 expected_recv=65492 expected_overflow=0 exact_recv_per_rank=[33096, 32731, 33096, 33096, 32758, 32740, 31531, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=34816 expected_recv=65499 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32731, 33096, 33096, 32758, 32740, 31531, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=34816 expected_recv=63009 expected_overflow=0 exact_recv_per_rank=[33096, 32731, 33096, 33096, 32758, 32740, 31531, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=34816 expected_recv=65520 expected_overflow=0 exact_recv_per_rank=[33096, 32731, 33096, 33096, 32758, 32740, 31531, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=34816 expected_recv=65492 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=34816 expected_recv=63009 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=34816 expected_recv=65520 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=64384 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=34816 expected_recv=65652 expected_overflow=0 exact_recv_per_rank=[32807, 32753, 33097, 32764, 32401, 33096, 32425, 32801] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=34816 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[32807, 32753, 33097, 32764, 32401, 33096, 32425, 32801] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32807, 32753, 33097, 32764, 32401, 33096, 32425, 32801] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=34816 expected_recv=65652 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32807, 32753, 33097, 32764, 32401, 33096, 32425, 32801] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=34816 expected_recv=65520 expected_overflow=0 exact_recv_per_rank=[32807, 32753, 33097, 32764, 32401, 33096, 32425, 32801] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=34816 expected_recv=65573 expected_overflow=0 exact_recv_per_rank=[32807, 32753, 33097, 32764, 32401, 33096, 32425, 32801] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=34816 expected_recv=64799 expected_overflow=0 exact_recv_per_rank=[32807, 32753, 33097, 32764, 32401, 33096, 32425, 32801] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=34816 expected_recv=65530 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=34816 expected_recv=64830 expected_overflow=0 exact_recv_per_rank=[32807, 32753, 33097, 32764, 32401, 33096, 32425, 32801] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=34816 expected_recv=65520 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=34816 expected_recv=65573 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=34816 expected_recv=64799 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=34816 expected_recv=64830 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=34816 expected_recv=65864 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32768, 33096, 31068, 33096, 32826] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=34816 expected_recv=65864 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32768, 33096, 31068, 33096, 32826] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=34816 expected_recv=65864 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=34816 expected_recv=62173 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32768, 33096, 31068, 33096, 32826] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32768, 33096, 31068, 33096, 32826] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=34816 expected_recv=66195 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32768, 33096, 31068, 33096, 32826] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=34816 expected_recv=65616 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32768, 33096, 31068, 33096, 32826] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=34816 expected_recv=65864 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32768, 33096, 31068, 33096, 32826] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=34816 expected_recv=62173 actual_valid_recv=62592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33096, 32768, 33096, 31068, 33096, 32826] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=34816 expected_recv=66195 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=34816 expected_recv=65616 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=62592 slack_slots=7040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=34816 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32838, 32772, 32050, 33096, 32781, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=34816 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32838, 32772, 32050, 33096, 32781, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=34816 expected_recv=65542 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32838, 32772, 32050, 33096, 32781, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=34816 expected_recv=65541 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32838, 32772, 32050, 33096, 32781, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=34816 expected_recv=65643 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32838, 32772, 32050, 33096, 32781, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=34816 expected_recv=65456 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32838, 32772, 32050, 33096, 32781, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=34816 expected_recv=65563 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32838, 32772, 32050, 33096, 32781, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=34816 expected_recv=64159 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32838, 32772, 32050, 33096, 32781, 32737] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=34816 expected_recv=65643 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=34816 expected_recv=65456 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=34816 expected_recv=65563 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=34816 expected_recv=64159 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32210, 32462, 32730, 33096, 33096, 32359] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65024 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=34816 expected_recv=65133 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32210, 32462, 32730, 33096, 33096, 32359] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=34816 expected_recv=65133 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32210, 32462, 32730, 33096, 33096, 32359] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32210, 32462, 32730, 33096, 33096, 32359] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=34816 expected_recv=64210 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32210, 32462, 32730, 33096, 33096, 32359] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=34816 expected_recv=64707 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32210, 32462, 32730, 33096, 33096, 32359] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=34816 expected_recv=65470 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32210, 32462, 32730, 33096, 33096, 32359] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32210, 32462, 32730, 33096, 33096, 32359] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=34816 expected_recv=64210 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=34816 expected_recv=64707 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=34816 expected_recv=65470 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=34816 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32760, 33096, 33096, 32758, 32462, 31779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65024 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=34816 expected_recv=66194 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32760, 33096, 33096, 32758, 32462, 31779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=34816 expected_recv=65518 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32760, 33096, 33096, 32758, 32462, 31779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=34816 expected_recv=65518 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32760, 33096, 33096, 32758, 32462, 31779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=34816 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32760, 33096, 33096, 32758, 32462, 31779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32760, 33096, 33096, 32758, 32462, 31779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=34816 expected_recv=63555 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32760, 33096, 33096, 32758, 32462, 31779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=34816 expected_recv=64919 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32760, 33096, 33096, 32758, 32462, 31779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=34816 expected_recv=65526 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=34816 expected_recv=63555 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=34816 expected_recv=64919 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=67200 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=64384 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=34816 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32482, 33096, 32759, 32768, 32410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=34816 expected_recv=64991 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32482, 33096, 32759, 32768, 32410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=34816 expected_recv=65533 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=34816 expected_recv=65524 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32482, 33096, 32759, 32768, 32410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=34816 expected_recv=64991 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=34816 expected_recv=65524 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32482, 33096, 32759, 32768, 32410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32482, 33096, 32759, 32768, 32410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=34816 expected_recv=65528 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32482, 33096, 32759, 32768, 32410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=34816 expected_recv=65171 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32482, 33096, 32759, 32768, 32410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=34816 expected_recv=65157 expected_overflow=0 exact_recv_per_rank=[32768, 32765, 33096, 32482, 33096, 32759, 32768, 32410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=34816 expected_recv=65528 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=34816 expected_recv=65171 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=34816 expected_recv=65157 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=34816 expected_recv=64803 expected_overflow=0 exact_recv_per_rank=[32382, 32764, 33096, 32772, 32698, 32771, 32565, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=34816 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[32382, 32764, 33096, 32772, 32698, 32771, 32565, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=34816 expected_recv=64803 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=34816 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[32382, 32764, 33096, 32772, 32698, 32771, 32565, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=34816 expected_recv=65540 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=34816 expected_recv=65540 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32382, 32764, 33096, 32772, 32698, 32771, 32565, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=34816 expected_recv=65277 expected_overflow=0 exact_recv_per_rank=[32382, 32764, 33096, 32772, 32698, 32771, 32565, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32382, 32764, 33096, 32772, 32698, 32771, 32565, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=34816 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[32382, 32764, 33096, 32772, 32698, 32771, 32565, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=34816 expected_recv=65212 expected_overflow=0 exact_recv_per_rank=[32382, 32764, 33096, 32772, 32698, 32771, 32565, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=34816 expected_recv=65277 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=34816 expected_recv=64309 expected_overflow=0 exact_recv_per_rank=[32115, 32758, 32660, 32227, 33098, 33096, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=34816 expected_recv=65532 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=34816 expected_recv=65212 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=34816 expected_recv=64415 expected_overflow=0 exact_recv_per_rank=[32115, 32758, 32660, 32227, 33098, 33096, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32115, 32758, 32660, 32227, 33098, 33096, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=34816 expected_recv=64309 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=34816 expected_recv=64415 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=34816 expected_recv=65277 expected_overflow=0 exact_recv_per_rank=[32115, 32758, 32660, 32227, 33098, 33096, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32115, 32758, 32660, 32227, 33098, 33096, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32115, 32758, 32660, 32227, 33098, 33096, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=34816 expected_recv=65277 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=34816 expected_recv=65519 expected_overflow=0 exact_recv_per_rank=[32115, 32758, 32660, 32227, 33098, 33096, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32115, 32758, 32660, 32227, 33098, 33096, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32435, 32430, 32465, 33096, 32772, 33096, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=34816 expected_recv=65519 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=34816 expected_recv=64959 expected_overflow=0 exact_recv_per_rank=[33096, 32435, 32430, 32465, 33096, 32772, 33096, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=34816 expected_recv=65500 expected_overflow=0 exact_recv_per_rank=[33096, 32435, 32430, 32465, 33096, 32772, 33096, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=34816 expected_recv=64959 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=34816 expected_recv=65500 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=34816 expected_recv=64872 expected_overflow=0 exact_recv_per_rank=[33096, 32435, 32430, 32465, 33096, 32772, 33096, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32435, 32430, 32465, 33096, 32772, 33096, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=34816 expected_recv=65502 expected_overflow=0 exact_recv_per_rank=[33096, 32435, 32430, 32465, 33096, 32772, 33096, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=34816 expected_recv=64872 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=34816 expected_recv=65502 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=34816 expected_recv=64879 expected_overflow=0 exact_recv_per_rank=[33096, 32435, 32430, 32465, 33096, 32772, 33096, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32435, 32430, 32465, 33096, 32772, 33096, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=34816 expected_recv=64827 expected_overflow=0 exact_recv_per_rank=[32405, 33096, 33096, 33096, 32463, 32768, 32459, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32405, 33096, 33096, 33096, 32463, 32768, 32459, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=34816 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[32405, 33096, 33096, 33096, 32463, 32768, 32459, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=34816 expected_recv=64879 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=34816 expected_recv=64827 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=34816 expected_recv=65535 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32405, 33096, 33096, 33096, 32463, 32768, 32459, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=34816 expected_recv=64915 expected_overflow=0 exact_recv_per_rank=[32405, 33096, 33096, 33096, 32463, 32768, 32459, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=34816 expected_recv=65528 expected_overflow=0 exact_recv_per_rank=[32405, 33096, 33096, 33096, 32463, 32768, 32459, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=34816 expected_recv=64915 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=34816 expected_recv=65528 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=34816 expected_recv=64910 expected_overflow=0 exact_recv_per_rank=[32467, 32815, 33096, 32100, 32751, 33096, 33064, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32405, 33096, 33096, 33096, 32463, 32768, 32459, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32467, 32815, 33096, 32100, 32751, 33096, 33064, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=34816 expected_recv=64907 expected_overflow=0 exact_recv_per_rank=[32405, 33096, 33096, 33096, 32463, 32768, 32459, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=34816 expected_recv=64413 expected_overflow=0 exact_recv_per_rank=[32467, 32815, 33096, 32100, 32751, 33096, 33064, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=34816 expected_recv=64910 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=34816 expected_recv=64907 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=34816 expected_recv=64413 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32467, 32815, 33096, 32100, 32751, 33096, 33064, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65024 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=34816 expected_recv=65506 expected_overflow=0 exact_recv_per_rank=[32467, 32815, 33096, 32100, 32751, 33096, 33064, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=34816 expected_recv=65522 expected_overflow=0 exact_recv_per_rank=[32467, 32815, 33096, 32100, 32751, 33096, 33064, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=34816 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[32241, 32569, 33096, 32760, 33096, 32778, 32765, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=34816 expected_recv=64642 expected_overflow=0 exact_recv_per_rank=[32241, 32569, 33096, 32760, 33096, 32778, 32765, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=34816 expected_recv=65589 expected_overflow=0 exact_recv_per_rank=[32467, 32815, 33096, 32100, 32751, 33096, 33064, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=34816 expected_recv=65506 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=34816 expected_recv=65522 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=34816 expected_recv=65546 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=34816 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[32241, 32569, 33096, 32760, 33096, 32778, 32765, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=34816 expected_recv=64642 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=34816 expected_recv=65964 expected_overflow=0 exact_recv_per_rank=[32467, 32815, 33096, 32100, 32751, 33096, 33064, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=34816 expected_recv=65589 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=34816 expected_recv=65527 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=34816 expected_recv=65964 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32241, 32569, 33096, 32760, 33096, 32778, 32765, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=34816 expected_recv=65640 expected_overflow=0 exact_recv_per_rank=[32241, 32569, 33096, 32760, 33096, 32778, 32765, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32241, 32569, 33096, 32760, 33096, 32778, 32765, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=34816 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[32185, 32765, 32774, 32775, 33096, 32764, 32689, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=34816 expected_recv=64319 expected_overflow=0 exact_recv_per_rank=[32185, 32765, 32774, 32775, 33096, 32764, 32689, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=34816 expected_recv=65640 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=34816 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[32185, 32765, 32774, 32775, 33096, 32764, 32689, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=34816 expected_recv=65532 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=34816 expected_recv=64319 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=34816 expected_recv=65016 expected_overflow=0 exact_recv_per_rank=[32241, 32569, 33096, 32760, 33096, 32778, 32765, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=34816 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[32241, 32569, 33096, 32760, 33096, 32778, 32765, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=34816 expected_recv=65544 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=34816 expected_recv=65016 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=34816 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[32185, 32765, 32774, 32775, 33096, 32764, 32689, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=34816 expected_recv=65533 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=34816 expected_recv=65540 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32185, 32765, 32774, 32775, 33096, 32764, 32689, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32185, 32765, 32774, 32775, 33096, 32764, 32689, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=34816 expected_recv=65389 expected_overflow=0 exact_recv_per_rank=[33096, 33092, 33096, 32776, 31953, 32641, 32767, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33092, 33096, 32776, 31953, 32641, 32767, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=34816 expected_recv=65389 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=34816 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[33096, 33092, 33096, 32776, 31953, 32641, 32767, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=34816 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[32185, 32765, 32774, 32775, 33096, 32764, 32689, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=34816 expected_recv=65436 expected_overflow=0 exact_recv_per_rank=[32185, 32765, 32774, 32775, 33096, 32764, 32689, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=34816 expected_recv=65543 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33092, 33096, 32776, 31953, 32641, 32767, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=34816 expected_recv=65533 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=34816 expected_recv=65436 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=34816 expected_recv=65479 expected_overflow=0 exact_recv_per_rank=[33096, 33092, 33096, 32776, 31953, 32641, 32767, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=34816 expected_recv=64101 expected_overflow=0 exact_recv_per_rank=[33096, 33092, 33096, 32776, 31953, 32641, 32767, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[26748, 22528, 14503, 25542, 12344, 25915, 55431, 79133] avg=32768.0000 max=79133 max_rank=7 min=12344 min_rank=4 imbalance_ratio=2.414948 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=34816 expected_recv=65479 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=34816 expected_recv=64101 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[26748, 22528, 14503, 25542, 12344, 25915, 55431, 79133] avg=32768.0000 max=79133 max_rank=7 min=12344 min_rank=4 imbalance_ratio=2.414948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[26748, 22528, 14503, 25542, 12344, 25915, 55431, 79133] avg=32768.0000 max=79133 max_rank=7 min=12344 min_rank=4 imbalance_ratio=2.414948 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=34816 expected_recv=65857 expected_overflow=0 exact_recv_per_rank=[33096, 33092, 33096, 32776, 31953, 32641, 32767, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=64768 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=34816 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[33096, 33092, 33096, 32776, 31953, 32641, 32767, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[26748, 22528, 14503, 25542, 12344, 25915, 55431, 79133] avg=32768.0000 max=79133 max_rank=7 min=12344 min_rank=4 imbalance_ratio=2.414948 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=34816 expected_recv=65857 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=34816 expected_recv=65535 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[33096, 30472, 33097, 33096, 33095, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=46036 max_load_reduction_pct=0.581755 moved_to_replicas=139188 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[33096, 30472, 33097, 33096, 33095, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=46036 max_load_reduction_pct=0.581755 moved_to_replicas=139188 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[33096, 30472, 33097, 33096, 33095, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=46036 max_load_reduction_pct=0.581755 moved_to_replicas=139188 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[33096, 30472, 33097, 33096, 33095, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=46036 max_load_reduction_pct=0.581755 moved_to_replicas=139188 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[26748, 22528, 14503, 25542, 12344, 25915, 55431, 79133] avg=32768.0000 max=79133 max_rank=7 min=12344 min_rank=4 imbalance_ratio=2.414948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[26748, 22528, 14503, 25542, 12344, 25915, 55431, 79133] avg=32768.0000 max=79133 max_rank=7 min=12344 min_rank=4 imbalance_ratio=2.414948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[33613, 32731, 35039, 93123, 32758, 6, 499, 34375] avg=32768.0000 max=93123 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.841888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[33613, 32731, 35039, 93123, 32758, 6, 499, 34375] avg=32768.0000 max=93123 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.841888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[26748, 22528, 14503, 25542, 12344, 25915, 55431, 79133] avg=32768.0000 max=79133 max_rank=7 min=12344 min_rank=4 imbalance_ratio=2.414948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[33613, 32731, 35039, 93123, 32758, 6, 499, 34375] avg=32768.0000 max=93123 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.841888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[26748, 22528, 14503, 25542, 12344, 25915, 55431, 79133] avg=32768.0000 max=79133 max_rank=7 min=12344 min_rank=4 imbalance_ratio=2.414948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[33096, 32731, 33096, 33096, 32758, 32740, 31531, 33096] avg=32768.0000 max=33096 max_rank=0 min=31531 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60027 max_load_reduction_pct=0.644599 moved_to_replicas=127587 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[33096, 30472, 33097, 33096, 33095, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=46036 max_load_reduction_pct=0.581755 moved_to_replicas=139188 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[33096, 30472, 33097, 33096, 33095, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=46036 max_load_reduction_pct=0.581755 moved_to_replicas=139188 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[33613, 32731, 35039, 93123, 32758, 6, 499, 34375] avg=32768.0000 max=93123 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.841888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[33096, 32731, 33096, 33096, 32758, 32740, 31531, 33096] avg=32768.0000 max=33096 max_rank=0 min=31531 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60027 max_load_reduction_pct=0.644599 moved_to_replicas=127587 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[33096, 32731, 33096, 33096, 32758, 32740, 31531, 33096] avg=32768.0000 max=33096 max_rank=0 min=31531 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60027 max_load_reduction_pct=0.644599 moved_to_replicas=127587 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[33096, 30472, 33097, 33096, 33095, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=46036 max_load_reduction_pct=0.581755 moved_to_replicas=139188 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[33096, 30472, 33097, 33096, 33095, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=46036 max_load_reduction_pct=0.581755 moved_to_replicas=139188 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32753, 37531, 32764, 18, 65479, 27991, 32801] avg=32768.0000 max=65479 max_rank=5 min=18 min_rank=4 imbalance_ratio=1.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[33096, 32731, 33096, 33096, 32758, 32740, 31531, 33096] avg=32768.0000 max=33096 max_rank=0 min=31531 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60027 max_load_reduction_pct=0.644599 moved_to_replicas=127587 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[33613, 32731, 35039, 93123, 32758, 6, 499, 34375] avg=32768.0000 max=93123 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.841888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[33613, 32731, 35039, 93123, 32758, 6, 499, 34375] avg=32768.0000 max=93123 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.841888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32753, 37531, 32764, 18, 65479, 27991, 32801] avg=32768.0000 max=65479 max_rank=5 min=18 min_rank=4 imbalance_ratio=1.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32753, 37531, 32764, 18, 65479, 27991, 32801] avg=32768.0000 max=65479 max_rank=5 min=18 min_rank=4 imbalance_ratio=1.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[33613, 32731, 35039, 93123, 32758, 6, 499, 34375] avg=32768.0000 max=93123 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.841888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32753, 33097, 32764, 32401, 33096, 32425, 32801] avg=32768.0000 max=33097 max_rank=2 min=32401 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32382 max_load_reduction_pct=0.494540 moved_to_replicas=74558 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[33613, 32731, 35039, 93123, 32758, 6, 499, 34375] avg=32768.0000 max=93123 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.841888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[33096, 32731, 33096, 33096, 32758, 32740, 31531, 33096] avg=32768.0000 max=33096 max_rank=0 min=31531 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60027 max_load_reduction_pct=0.644599 moved_to_replicas=127587 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[33096, 32731, 33096, 33096, 32758, 32740, 31531, 33096] avg=32768.0000 max=33096 max_rank=0 min=31531 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60027 max_load_reduction_pct=0.644599 moved_to_replicas=127587 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32753, 37531, 32764, 18, 65479, 27991, 32801] avg=32768.0000 max=65479 max_rank=5 min=18 min_rank=4 imbalance_ratio=1.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32753, 33097, 32764, 32401, 33096, 32425, 32801] avg=32768.0000 max=33097 max_rank=2 min=32401 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32382 max_load_reduction_pct=0.494540 moved_to_replicas=74558 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32753, 33097, 32764, 32401, 33096, 32425, 32801] avg=32768.0000 max=33097 max_rank=2 min=32401 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32382 max_load_reduction_pct=0.494540 moved_to_replicas=74558 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[33096, 32731, 33096, 33096, 32758, 32740, 31531, 33096] avg=32768.0000 max=33096 max_rank=0 min=31531 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60027 max_load_reduction_pct=0.644599 moved_to_replicas=127587 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[33096, 32731, 33096, 33096, 32758, 32740, 31531, 33096] avg=32768.0000 max=33096 max_rank=0 min=31531 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60027 max_load_reduction_pct=0.644599 moved_to_replicas=127587 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[9, 5351, 65441, 6, 65516, 24872, 68123, 32826] avg=32768.0000 max=68123 max_rank=6 min=6 min_rank=3 imbalance_ratio=2.078949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32753, 33097, 32764, 32401, 33096, 32425, 32801] avg=32768.0000 max=33097 max_rank=2 min=32401 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32382 max_load_reduction_pct=0.494540 moved_to_replicas=74558 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32753, 37531, 32764, 18, 65479, 27991, 32801] avg=32768.0000 max=65479 max_rank=5 min=18 min_rank=4 imbalance_ratio=1.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32753, 37531, 32764, 18, 65479, 27991, 32801] avg=32768.0000 max=65479 max_rank=5 min=18 min_rank=4 imbalance_ratio=1.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[9, 5351, 65441, 6, 65516, 24872, 68123, 32826] avg=32768.0000 max=68123 max_rank=6 min=6 min_rank=3 imbalance_ratio=2.078949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[33096, 33098, 33096, 32768, 33096, 31068, 33096, 32826] avg=32768.0000 max=33098 max_rank=1 min=31068 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=35025 max_load_reduction_pct=0.514144 moved_to_replicas=199513 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[9, 5351, 65441, 6, 65516, 24872, 68123, 32826] avg=32768.0000 max=68123 max_rank=6 min=6 min_rank=3 imbalance_ratio=2.078949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32753, 37531, 32764, 18, 65479, 27991, 32801] avg=32768.0000 max=65479 max_rank=5 min=18 min_rank=4 imbalance_ratio=1.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32753, 37531, 32764, 18, 65479, 27991, 32801] avg=32768.0000 max=65479 max_rank=5 min=18 min_rank=4 imbalance_ratio=1.998260 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32753, 33097, 32764, 32401, 33096, 32425, 32801] avg=32768.0000 max=33097 max_rank=2 min=32401 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32382 max_load_reduction_pct=0.494540 moved_to_replicas=74558 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32753, 33097, 32764, 32401, 33096, 32425, 32801] avg=32768.0000 max=33097 max_rank=2 min=32401 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32382 max_load_reduction_pct=0.494540 moved_to_replicas=74558 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[33096, 33098, 33096, 32768, 33096, 31068, 33096, 32826] avg=32768.0000 max=33098 max_rank=1 min=31068 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=35025 max_load_reduction_pct=0.514144 moved_to_replicas=199513 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[9, 5351, 65441, 6, 65516, 24872, 68123, 32826] avg=32768.0000 max=68123 max_rank=6 min=6 min_rank=3 imbalance_ratio=2.078949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[33096, 33098, 33096, 32768, 33096, 31068, 33096, 32826] avg=32768.0000 max=33098 max_rank=1 min=31068 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=35025 max_load_reduction_pct=0.514144 moved_to_replicas=199513 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32753, 33097, 32764, 32401, 33096, 32425, 32801] avg=32768.0000 max=33097 max_rank=2 min=32401 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32382 max_load_reduction_pct=0.494540 moved_to_replicas=74558 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32753, 33097, 32764, 32401, 33096, 32425, 32801] avg=32768.0000 max=33097 max_rank=2 min=32401 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32382 max_load_reduction_pct=0.494540 moved_to_replicas=74558 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[11, 69712, 32838, 32772, 19, 61274, 32781, 32737] avg=32768.0000 max=69712 max_rank=1 min=11 min_rank=0 imbalance_ratio=2.127441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[33096, 33098, 33096, 32768, 33096, 31068, 33096, 32826] avg=32768.0000 max=33098 max_rank=1 min=31068 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=35025 max_load_reduction_pct=0.514144 moved_to_replicas=199513 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[9, 5351, 65441, 6, 65516, 24872, 68123, 32826] avg=32768.0000 max=68123 max_rank=6 min=6 min_rank=3 imbalance_ratio=2.078949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[9, 5351, 65441, 6, 65516, 24872, 68123, 32826] avg=32768.0000 max=68123 max_rank=6 min=6 min_rank=3 imbalance_ratio=2.078949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[11, 69712, 32838, 32772, 19, 61274, 32781, 32737] avg=32768.0000 max=69712 max_rank=1 min=11 min_rank=0 imbalance_ratio=2.127441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[32774, 33096, 32838, 32772, 32050, 33096, 32781, 32737] avg=32768.0000 max=33096 max_rank=1 min=32050 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=36616 max_load_reduction_pct=0.525247 moved_to_replicas=129663 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[11, 69712, 32838, 32772, 19, 61274, 32781, 32737] avg=32768.0000 max=69712 max_rank=1 min=11 min_rank=0 imbalance_ratio=2.127441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[9, 5351, 65441, 6, 65516, 24872, 68123, 32826] avg=32768.0000 max=68123 max_rank=6 min=6 min_rank=3 imbalance_ratio=2.078949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[33096, 33098, 33096, 32768, 33096, 31068, 33096, 32826] avg=32768.0000 max=33098 max_rank=1 min=31068 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=35025 max_load_reduction_pct=0.514144 moved_to_replicas=199513 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[9, 5351, 65441, 6, 65516, 24872, 68123, 32826] avg=32768.0000 max=68123 max_rank=6 min=6 min_rank=3 imbalance_ratio=2.078949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[33096, 33098, 33096, 32768, 33096, 31068, 33096, 32826] avg=32768.0000 max=33098 max_rank=1 min=31068 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=35025 max_load_reduction_pct=0.514144 moved_to_replicas=199513 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[32774, 33096, 32838, 32772, 32050, 33096, 32781, 32737] avg=32768.0000 max=33096 max_rank=1 min=32050 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=36616 max_load_reduction_pct=0.525247 moved_to_replicas=129663 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[11, 69712, 32838, 32772, 19, 61274, 32781, 32737] avg=32768.0000 max=69712 max_rank=1 min=11 min_rank=0 imbalance_ratio=2.127441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[32774, 33096, 32838, 32772, 32050, 33096, 32781, 32737] avg=32768.0000 max=33096 max_rank=1 min=32050 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=36616 max_load_reduction_pct=0.525247 moved_to_replicas=129663 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[33096, 33098, 33096, 32768, 33096, 31068, 33096, 32826] avg=32768.0000 max=33098 max_rank=1 min=31068 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=35025 max_load_reduction_pct=0.514144 moved_to_replicas=199513 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[56702, 41818, 24, 133, 32730, 65282, 65449, 6] avg=32768.0000 max=65449 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.997345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[33096, 33098, 33096, 32768, 33096, 31068, 33096, 32826] avg=32768.0000 max=33098 max_rank=1 min=31068 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=35025 max_load_reduction_pct=0.514144 moved_to_replicas=199513 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[32774, 33096, 32838, 32772, 32050, 33096, 32781, 32737] avg=32768.0000 max=33096 max_rank=1 min=32050 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=36616 max_load_reduction_pct=0.525247 moved_to_replicas=129663 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[11, 69712, 32838, 32772, 19, 61274, 32781, 32737] avg=32768.0000 max=69712 max_rank=1 min=11 min_rank=0 imbalance_ratio=2.127441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[11, 69712, 32838, 32772, 19, 61274, 32781, 32737] avg=32768.0000 max=69712 max_rank=1 min=11 min_rank=0 imbalance_ratio=2.127441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[56702, 41818, 24, 133, 32730, 65282, 65449, 6] avg=32768.0000 max=65449 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.997345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[33096, 33095, 32210, 32462, 32730, 33096, 33096, 32359] avg=32768.0000 max=33096 max_rank=0 min=32210 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=193648 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[56702, 41818, 24, 133, 32730, 65282, 65449, 6] avg=32768.0000 max=65449 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.997345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[32774, 33096, 32838, 32772, 32050, 33096, 32781, 32737] avg=32768.0000 max=33096 max_rank=1 min=32050 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=36616 max_load_reduction_pct=0.525247 moved_to_replicas=129663 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[11, 69712, 32838, 32772, 19, 61274, 32781, 32737] avg=32768.0000 max=69712 max_rank=1 min=11 min_rank=0 imbalance_ratio=2.127441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[11, 69712, 32838, 32772, 19, 61274, 32781, 32737] avg=32768.0000 max=69712 max_rank=1 min=11 min_rank=0 imbalance_ratio=2.127441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[32774, 33096, 32838, 32772, 32050, 33096, 32781, 32737] avg=32768.0000 max=33096 max_rank=1 min=32050 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=36616 max_load_reduction_pct=0.525247 moved_to_replicas=129663 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[33096, 33095, 32210, 32462, 32730, 33096, 33096, 32359] avg=32768.0000 max=33096 max_rank=0 min=32210 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=193648 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[56702, 41818, 24, 133, 32730, 65282, 65449, 6] avg=32768.0000 max=65449 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.997345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[33096, 33095, 32210, 32462, 32730, 33096, 33096, 32359] avg=32768.0000 max=33096 max_rank=0 min=32210 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=193648 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[45765, 818, 32760, 65540, 65513, 32758, 18, 18972] avg=32768.0000 max=65540 max_rank=3 min=18 min_rank=6 imbalance_ratio=2.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[32774, 33096, 32838, 32772, 32050, 33096, 32781, 32737] avg=32768.0000 max=33096 max_rank=1 min=32050 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=36616 max_load_reduction_pct=0.525247 moved_to_replicas=129663 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[32774, 33096, 32838, 32772, 32050, 33096, 32781, 32737] avg=32768.0000 max=33096 max_rank=1 min=32050 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=36616 max_load_reduction_pct=0.525247 moved_to_replicas=129663 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[33096, 33095, 32210, 32462, 32730, 33096, 33096, 32359] avg=32768.0000 max=33096 max_rank=0 min=32210 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=193648 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[56702, 41818, 24, 133, 32730, 65282, 65449, 6] avg=32768.0000 max=65449 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.997345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[56702, 41818, 24, 133, 32730, 65282, 65449, 6] avg=32768.0000 max=65449 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.997345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[33097, 33096, 32760, 33096, 33096, 32758, 32462, 31779] avg=32768.0000 max=33097 max_rank=0 min=31779 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32443 max_load_reduction_pct=0.495011 moved_to_replicas=155179 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[45765, 818, 32760, 65540, 65513, 32758, 18, 18972] avg=32768.0000 max=65540 max_rank=3 min=18 min_rank=6 imbalance_ratio=2.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[45765, 818, 32760, 65540, 65513, 32758, 18, 18972] avg=32768.0000 max=65540 max_rank=3 min=18 min_rank=6 imbalance_ratio=2.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[33096, 33095, 32210, 32462, 32730, 33096, 33096, 32359] avg=32768.0000 max=33096 max_rank=0 min=32210 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=193648 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[56702, 41818, 24, 133, 32730, 65282, 65449, 6] avg=32768.0000 max=65449 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.997345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[33096, 33095, 32210, 32462, 32730, 33096, 33096, 32359] avg=32768.0000 max=33096 max_rank=0 min=32210 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=193648 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[56702, 41818, 24, 133, 32730, 65282, 65449, 6] avg=32768.0000 max=65449 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.997345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[33097, 33096, 32760, 33096, 33096, 32758, 32462, 31779] avg=32768.0000 max=33097 max_rank=0 min=31779 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32443 max_load_reduction_pct=0.495011 moved_to_replicas=155179 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[45765, 818, 32760, 65540, 65513, 32758, 18, 18972] avg=32768.0000 max=65540 max_rank=3 min=18 min_rank=6 imbalance_ratio=2.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[33097, 33096, 32760, 33096, 33096, 32758, 32462, 31779] avg=32768.0000 max=33097 max_rank=0 min=31779 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32443 max_load_reduction_pct=0.495011 moved_to_replicas=155179 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32765, 98255, 53, 65525, 32759, 7, 12] avg=32768.0000 max=98255 max_rank=2 min=7 min_rank=6 imbalance_ratio=2.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[33096, 33095, 32210, 32462, 32730, 33096, 33096, 32359] avg=32768.0000 max=33096 max_rank=0 min=32210 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=193648 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[33096, 33095, 32210, 32462, 32730, 33096, 33096, 32359] avg=32768.0000 max=33096 max_rank=0 min=32210 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32353 max_load_reduction_pct=0.494324 moved_to_replicas=193648 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[33097, 33096, 32760, 33096, 33096, 32758, 32462, 31779] avg=32768.0000 max=33097 max_rank=0 min=31779 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32443 max_load_reduction_pct=0.495011 moved_to_replicas=155179 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32765, 33096, 32482, 33096, 32759, 32768, 32410] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65159 max_load_reduction_pct=0.663162 moved_to_replicas=195174 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[45765, 818, 32760, 65540, 65513, 32758, 18, 18972] avg=32768.0000 max=65540 max_rank=3 min=18 min_rank=6 imbalance_ratio=2.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[45765, 818, 32760, 65540, 65513, 32758, 18, 18972] avg=32768.0000 max=65540 max_rank=3 min=18 min_rank=6 imbalance_ratio=2.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32765, 98255, 53, 65525, 32759, 7, 12] avg=32768.0000 max=98255 max_rank=2 min=7 min_rank=6 imbalance_ratio=2.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32765, 98255, 53, 65525, 32759, 7, 12] avg=32768.0000 max=98255 max_rank=2 min=7 min_rank=6 imbalance_ratio=2.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[33097, 33096, 32760, 33096, 33096, 32758, 32462, 31779] avg=32768.0000 max=33097 max_rank=0 min=31779 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32443 max_load_reduction_pct=0.495011 moved_to_replicas=155179 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[45765, 818, 32760, 65540, 65513, 32758, 18, 18972] avg=32768.0000 max=65540 max_rank=3 min=18 min_rank=6 imbalance_ratio=2.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[33097, 33096, 32760, 33096, 33096, 32758, 32462, 31779] avg=32768.0000 max=33097 max_rank=0 min=31779 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32443 max_load_reduction_pct=0.495011 moved_to_replicas=155179 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32765, 33096, 32482, 33096, 32759, 32768, 32410] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65159 max_load_reduction_pct=0.663162 moved_to_replicas=195174 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[45765, 818, 32760, 65540, 65513, 32758, 18, 18972] avg=32768.0000 max=65540 max_rank=3 min=18 min_rank=6 imbalance_ratio=2.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[324, 32764, 98283, 11, 272, 32771, 32565, 65154] avg=32768.0000 max=98283 max_rank=2 min=11 min_rank=3 imbalance_ratio=2.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32765, 33096, 32482, 33096, 32759, 32768, 32410] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65159 max_load_reduction_pct=0.663162 moved_to_replicas=195174 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32765, 98255, 53, 65525, 32759, 7, 12] avg=32768.0000 max=98255 max_rank=2 min=7 min_rank=6 imbalance_ratio=2.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[33097, 33096, 32760, 33096, 33096, 32758, 32462, 31779] avg=32768.0000 max=33097 max_rank=0 min=31779 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32443 max_load_reduction_pct=0.495011 moved_to_replicas=155179 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[32382, 32764, 33096, 32772, 32698, 32771, 32565, 33096] avg=32768.0000 max=33096 max_rank=2 min=32382 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65187 max_load_reduction_pct=0.663258 moved_to_replicas=194630 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[33097, 33096, 32760, 33096, 33096, 32758, 32462, 31779] avg=32768.0000 max=33097 max_rank=0 min=31779 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=32443 max_load_reduction_pct=0.495011 moved_to_replicas=155179 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32765, 33096, 32482, 33096, 32759, 32768, 32410] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65159 max_load_reduction_pct=0.663162 moved_to_replicas=195174 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32765, 98255, 53, 65525, 32759, 7, 12] avg=32768.0000 max=98255 max_rank=2 min=7 min_rank=6 imbalance_ratio=2.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32765, 98255, 53, 65525, 32759, 7, 12] avg=32768.0000 max=98255 max_rank=2 min=7 min_rank=6 imbalance_ratio=2.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[324, 32764, 98283, 11, 272, 32771, 32565, 65154] avg=32768.0000 max=98283 max_rank=2 min=11 min_rank=3 imbalance_ratio=2.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[324, 32764, 98283, 11, 272, 32771, 32565, 65154] avg=32768.0000 max=98283 max_rank=2 min=11 min_rank=3 imbalance_ratio=2.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32765, 98255, 53, 65525, 32759, 7, 12] avg=32768.0000 max=98255 max_rank=2 min=7 min_rank=6 imbalance_ratio=2.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32765, 33096, 32482, 33096, 32759, 32768, 32410] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65159 max_load_reduction_pct=0.663162 moved_to_replicas=195174 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32765, 33096, 32482, 33096, 32759, 32768, 32410] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65159 max_load_reduction_pct=0.663162 moved_to_replicas=195174 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[32382, 32764, 33096, 32772, 32698, 32771, 32565, 33096] avg=32768.0000 max=33096 max_rank=2 min=32382 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65187 max_load_reduction_pct=0.663258 moved_to_replicas=194630 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[102, 8, 32660, 32101, 21913, 66560, 64395, 44405] avg=32768.0000 max=66560 max_rank=5 min=8 min_rank=1 imbalance_ratio=2.031250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[32382, 32764, 33096, 32772, 32698, 32771, 32565, 33096] avg=32768.0000 max=33096 max_rank=2 min=32382 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65187 max_load_reduction_pct=0.663258 moved_to_replicas=194630 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32765, 98255, 53, 65525, 32759, 7, 12] avg=32768.0000 max=98255 max_rank=2 min=7 min_rank=6 imbalance_ratio=2.998505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[324, 32764, 98283, 11, 272, 32771, 32565, 65154] avg=32768.0000 max=98283 max_rank=2 min=11 min_rank=3 imbalance_ratio=2.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32765, 33096, 32482, 33096, 32759, 32768, 32410] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65159 max_load_reduction_pct=0.663162 moved_to_replicas=195174 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[32115, 32758, 32660, 32227, 33098, 33096, 33096, 33094] avg=32768.0000 max=33098 max_rank=4 min=32115 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=33462 max_load_reduction_pct=0.502734 moved_to_replicas=152552 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[32382, 32764, 33096, 32772, 32698, 32771, 32565, 33096] avg=32768.0000 max=33096 max_rank=2 min=32382 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65187 max_load_reduction_pct=0.663258 moved_to_replicas=194630 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32765, 33096, 32482, 33096, 32759, 32768, 32410] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65159 max_load_reduction_pct=0.663162 moved_to_replicas=195174 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[324, 32764, 98283, 11, 272, 32771, 32565, 65154] avg=32768.0000 max=98283 max_rank=2 min=11 min_rank=3 imbalance_ratio=2.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[324, 32764, 98283, 11, 272, 32771, 32565, 65154] avg=32768.0000 max=98283 max_rank=2 min=11 min_rank=3 imbalance_ratio=2.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[102, 8, 32660, 32101, 21913, 66560, 64395, 44405] avg=32768.0000 max=66560 max_rank=5 min=8 min_rank=1 imbalance_ratio=2.031250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[102, 8, 32660, 32101, 21913, 66560, 64395, 44405] avg=32768.0000 max=66560 max_rank=5 min=8 min_rank=1 imbalance_ratio=2.031250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[324, 32764, 98283, 11, 272, 32771, 32565, 65154] avg=32768.0000 max=98283 max_rank=2 min=11 min_rank=3 imbalance_ratio=2.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[32382, 32764, 33096, 32772, 32698, 32771, 32565, 33096] avg=32768.0000 max=33096 max_rank=2 min=32382 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65187 max_load_reduction_pct=0.663258 moved_to_replicas=194630 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[65436, 5, 12, 125, 65526, 32772, 65514, 32754] avg=32768.0000 max=65526 max_rank=4 min=5 min_rank=1 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[32382, 32764, 33096, 32772, 32698, 32771, 32565, 33096] avg=32768.0000 max=33096 max_rank=2 min=32382 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65187 max_load_reduction_pct=0.663258 moved_to_replicas=194630 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[32115, 32758, 32660, 32227, 33098, 33096, 33096, 33094] avg=32768.0000 max=33098 max_rank=4 min=32115 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=33462 max_load_reduction_pct=0.502734 moved_to_replicas=152552 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[32115, 32758, 32660, 32227, 33098, 33096, 33096, 33094] avg=32768.0000 max=33098 max_rank=4 min=32115 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=33462 max_load_reduction_pct=0.502734 moved_to_replicas=152552 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[102, 8, 32660, 32101, 21913, 66560, 64395, 44405] avg=32768.0000 max=66560 max_rank=5 min=8 min_rank=1 imbalance_ratio=2.031250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[324, 32764, 98283, 11, 272, 32771, 32565, 65154] avg=32768.0000 max=98283 max_rank=2 min=11 min_rank=3 imbalance_ratio=2.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[32382, 32764, 33096, 32772, 32698, 32771, 32565, 33096] avg=32768.0000 max=33096 max_rank=2 min=32382 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65187 max_load_reduction_pct=0.663258 moved_to_replicas=194630 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[33096, 32435, 32430, 32465, 33096, 32772, 33096, 32754] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=194503 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[32115, 32758, 32660, 32227, 33098, 33096, 33096, 33094] avg=32768.0000 max=33098 max_rank=4 min=32115 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=33462 max_load_reduction_pct=0.502734 moved_to_replicas=152552 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[102, 8, 32660, 32101, 21913, 66560, 64395, 44405] avg=32768.0000 max=66560 max_rank=5 min=8 min_rank=1 imbalance_ratio=2.031250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[32382, 32764, 33096, 32772, 32698, 32771, 32565, 33096] avg=32768.0000 max=33096 max_rank=2 min=32382 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65187 max_load_reduction_pct=0.663258 moved_to_replicas=194630 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[65436, 5, 12, 125, 65526, 32772, 65514, 32754] avg=32768.0000 max=65526 max_rank=4 min=5 min_rank=1 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[102, 8, 32660, 32101, 21913, 66560, 64395, 44405] avg=32768.0000 max=66560 max_rank=5 min=8 min_rank=1 imbalance_ratio=2.031250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[65436, 5, 12, 125, 65526, 32772, 65514, 32754] avg=32768.0000 max=65526 max_rank=4 min=5 min_rank=1 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[102, 8, 32660, 32101, 21913, 66560, 64395, 44405] avg=32768.0000 max=66560 max_rank=5 min=8 min_rank=1 imbalance_ratio=2.031250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[1381, 65447, 64120, 65538, 112, 32768, 17, 32761] avg=32768.0000 max=65538 max_rank=3 min=17 min_rank=6 imbalance_ratio=2.000061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[32115, 32758, 32660, 32227, 33098, 33096, 33096, 33094] avg=32768.0000 max=33098 max_rank=4 min=32115 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=33462 max_load_reduction_pct=0.502734 moved_to_replicas=152552 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[33096, 32435, 32430, 32465, 33096, 32772, 33096, 32754] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=194503 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[65436, 5, 12, 125, 65526, 32772, 65514, 32754] avg=32768.0000 max=65526 max_rank=4 min=5 min_rank=1 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[32115, 32758, 32660, 32227, 33098, 33096, 33096, 33094] avg=32768.0000 max=33098 max_rank=4 min=32115 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=33462 max_load_reduction_pct=0.502734 moved_to_replicas=152552 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[33096, 32435, 32430, 32465, 33096, 32772, 33096, 32754] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=194503 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[102, 8, 32660, 32101, 21913, 66560, 64395, 44405] avg=32768.0000 max=66560 max_rank=5 min=8 min_rank=1 imbalance_ratio=2.031250 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[32405, 33096, 33096, 33096, 32463, 32768, 32459, 32761] avg=32768.0000 max=33096 max_rank=1 min=32405 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32442 max_load_reduction_pct=0.495011 moved_to_replicas=191222 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[32115, 32758, 32660, 32227, 33098, 33096, 33096, 33094] avg=32768.0000 max=33098 max_rank=4 min=32115 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=33462 max_load_reduction_pct=0.502734 moved_to_replicas=152552 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[33096, 32435, 32430, 32465, 33096, 32772, 33096, 32754] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=194503 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[65436, 5, 12, 125, 65526, 32772, 65514, 32754] avg=32768.0000 max=65526 max_rank=4 min=5 min_rank=1 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[32115, 32758, 32660, 32227, 33098, 33096, 33096, 33094] avg=32768.0000 max=33098 max_rank=4 min=32115 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=33462 max_load_reduction_pct=0.502734 moved_to_replicas=152552 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[1381, 65447, 64120, 65538, 112, 32768, 17, 32761] avg=32768.0000 max=65538 max_rank=3 min=17 min_rank=6 imbalance_ratio=2.000061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[1381, 65447, 64120, 65538, 112, 32768, 17, 32761] avg=32768.0000 max=65538 max_rank=3 min=17 min_rank=6 imbalance_ratio=2.000061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[65436, 5, 12, 125, 65526, 32772, 65514, 32754] avg=32768.0000 max=65526 max_rank=4 min=5 min_rank=1 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[38, 32815, 65525, 141, 32751, 65055, 33064, 32755] avg=32768.0000 max=65525 max_rank=2 min=38 min_rank=0 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[65436, 5, 12, 125, 65526, 32772, 65514, 32754] avg=32768.0000 max=65526 max_rank=4 min=5 min_rank=1 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[33096, 32435, 32430, 32465, 33096, 32772, 33096, 32754] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=194503 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[32405, 33096, 33096, 33096, 32463, 32768, 32459, 32761] avg=32768.0000 max=33096 max_rank=1 min=32405 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32442 max_load_reduction_pct=0.495011 moved_to_replicas=191222 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[32405, 33096, 33096, 33096, 32463, 32768, 32459, 32761] avg=32768.0000 max=33096 max_rank=1 min=32405 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32442 max_load_reduction_pct=0.495011 moved_to_replicas=191222 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[1381, 65447, 64120, 65538, 112, 32768, 17, 32761] avg=32768.0000 max=65538 max_rank=3 min=17 min_rank=6 imbalance_ratio=2.000061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[33096, 32435, 32430, 32465, 33096, 32772, 33096, 32754] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=194503 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[32467, 32815, 33096, 32100, 32751, 33096, 33064, 32755] avg=32768.0000 max=33096 max_rank=2 min=32100 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=128658 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[65436, 5, 12, 125, 65526, 32772, 65514, 32754] avg=32768.0000 max=65526 max_rank=4 min=5 min_rank=1 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[33096, 32435, 32430, 32465, 33096, 32772, 33096, 32754] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=194503 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[32405, 33096, 33096, 33096, 32463, 32768, 32459, 32761] avg=32768.0000 max=33096 max_rank=1 min=32405 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32442 max_load_reduction_pct=0.495011 moved_to_replicas=191222 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[1381, 65447, 64120, 65538, 112, 32768, 17, 32761] avg=32768.0000 max=65538 max_rank=3 min=17 min_rank=6 imbalance_ratio=2.000061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[38, 32815, 65525, 141, 32751, 65055, 33064, 32755] avg=32768.0000 max=65525 max_rank=2 min=38 min_rank=0 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[33096, 32435, 32430, 32465, 33096, 32772, 33096, 32754] avg=32768.0000 max=33096 max_rank=0 min=32430 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=194503 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[38, 32815, 65525, 141, 32751, 65055, 33064, 32755] avg=32768.0000 max=65525 max_rank=2 min=38 min_rank=0 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[1381, 65447, 64120, 65538, 112, 32768, 17, 32761] avg=32768.0000 max=65538 max_rank=3 min=17 min_rank=6 imbalance_ratio=2.000061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[16, 89, 65576, 11, 98070, 32778, 32765, 32839] avg=32768.0000 max=98070 max_rank=4 min=11 min_rank=3 imbalance_ratio=2.992859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[1381, 65447, 64120, 65538, 112, 32768, 17, 32761] avg=32768.0000 max=65538 max_rank=3 min=17 min_rank=6 imbalance_ratio=2.000061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[32405, 33096, 33096, 33096, 32463, 32768, 32459, 32761] avg=32768.0000 max=33096 max_rank=1 min=32405 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32442 max_load_reduction_pct=0.495011 moved_to_replicas=191222 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[32467, 32815, 33096, 32100, 32751, 33096, 33064, 32755] avg=32768.0000 max=33096 max_rank=2 min=32100 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=128658 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[32467, 32815, 33096, 32100, 32751, 33096, 33064, 32755] avg=32768.0000 max=33096 max_rank=2 min=32100 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=128658 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[38, 32815, 65525, 141, 32751, 65055, 33064, 32755] avg=32768.0000 max=65525 max_rank=2 min=38 min_rank=0 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[32405, 33096, 33096, 33096, 32463, 32768, 32459, 32761] avg=32768.0000 max=33096 max_rank=1 min=32405 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32442 max_load_reduction_pct=0.495011 moved_to_replicas=191222 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[1381, 65447, 64120, 65538, 112, 32768, 17, 32761] avg=32768.0000 max=65538 max_rank=3 min=17 min_rank=6 imbalance_ratio=2.000061 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[32241, 32569, 33096, 32760, 33096, 32778, 32765, 32839] avg=32768.0000 max=33096 max_rank=2 min=32241 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64974 max_load_reduction_pct=0.662527 moved_to_replicas=195044 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[32405, 33096, 33096, 33096, 32463, 32768, 32459, 32761] avg=32768.0000 max=33096 max_rank=1 min=32405 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32442 max_load_reduction_pct=0.495011 moved_to_replicas=191222 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[38, 32815, 65525, 141, 32751, 65055, 33064, 32755] avg=32768.0000 max=65525 max_rank=2 min=38 min_rank=0 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[16, 89, 65576, 11, 98070, 32778, 32765, 32839] avg=32768.0000 max=98070 max_rank=4 min=11 min_rank=3 imbalance_ratio=2.992859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[32467, 32815, 33096, 32100, 32751, 33096, 33064, 32755] avg=32768.0000 max=33096 max_rank=2 min=32100 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=128658 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[16, 89, 65576, 11, 98070, 32778, 32765, 32839] avg=32768.0000 max=98070 max_rank=4 min=11 min_rank=3 imbalance_ratio=2.992859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[32405, 33096, 33096, 33096, 32463, 32768, 32459, 32761] avg=32768.0000 max=33096 max_rank=1 min=32405 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32442 max_load_reduction_pct=0.495011 moved_to_replicas=191222 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[38, 32815, 65525, 141, 32751, 65055, 33064, 32755] avg=32768.0000 max=65525 max_rank=2 min=38 min_rank=0 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[36, 32765, 32774, 32775, 63620, 32764, 32689, 34721] avg=32768.0000 max=63620 max_rank=4 min=36 min_rank=0 imbalance_ratio=1.941528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[38, 32815, 65525, 141, 32751, 65055, 33064, 32755] avg=32768.0000 max=65525 max_rank=2 min=38 min_rank=0 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[32467, 32815, 33096, 32100, 32751, 33096, 33064, 32755] avg=32768.0000 max=33096 max_rank=2 min=32100 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=128658 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[32241, 32569, 33096, 32760, 33096, 32778, 32765, 32839] avg=32768.0000 max=33096 max_rank=2 min=32241 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64974 max_load_reduction_pct=0.662527 moved_to_replicas=195044 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[32241, 32569, 33096, 32760, 33096, 32778, 32765, 32839] avg=32768.0000 max=33096 max_rank=2 min=32241 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64974 max_load_reduction_pct=0.662527 moved_to_replicas=195044 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[16, 89, 65576, 11, 98070, 32778, 32765, 32839] avg=32768.0000 max=98070 max_rank=4 min=11 min_rank=3 imbalance_ratio=2.992859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[32467, 32815, 33096, 32100, 32751, 33096, 33064, 32755] avg=32768.0000 max=33096 max_rank=2 min=32100 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=128658 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[32185, 32765, 32774, 32775, 33096, 32764, 32689, 33096] avg=32768.0000 max=33096 max_rank=4 min=32185 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30524 max_load_reduction_pct=0.479786 moved_to_replicas=64278 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[38, 32815, 65525, 141, 32751, 65055, 33064, 32755] avg=32768.0000 max=65525 max_rank=2 min=38 min_rank=0 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[32467, 32815, 33096, 32100, 32751, 33096, 33064, 32755] avg=32768.0000 max=33096 max_rank=2 min=32100 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=128658 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[16, 89, 65576, 11, 98070, 32778, 32765, 32839] avg=32768.0000 max=98070 max_rank=4 min=11 min_rank=3 imbalance_ratio=2.992859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[32241, 32569, 33096, 32760, 33096, 32778, 32765, 32839] avg=32768.0000 max=33096 max_rank=2 min=32241 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64974 max_load_reduction_pct=0.662527 moved_to_replicas=195044 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[36, 32765, 32774, 32775, 63620, 32764, 32689, 34721] avg=32768.0000 max=63620 max_rank=4 min=36 min_rank=0 imbalance_ratio=1.941528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[36, 32765, 32774, 32775, 63620, 32764, 32689, 34721] avg=32768.0000 max=63620 max_rank=4 min=36 min_rank=0 imbalance_ratio=1.941528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[32467, 32815, 33096, 32100, 32751, 33096, 33064, 32755] avg=32768.0000 max=33096 max_rank=2 min=32100 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=128658 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[16, 89, 65576, 11, 98070, 32778, 32765, 32839] avg=32768.0000 max=98070 max_rank=4 min=11 min_rank=3 imbalance_ratio=2.992859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[97755, 692, 65496, 32776, 17, 32641, 32767, 0] avg=32768.0000 max=97755 max_rank=0 min=0 min_rank=7 imbalance_ratio=2.983246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[16, 89, 65576, 11, 98070, 32778, 32765, 32839] avg=32768.0000 max=98070 max_rank=4 min=11 min_rank=3 imbalance_ratio=2.992859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[32241, 32569, 33096, 32760, 33096, 32778, 32765, 32839] avg=32768.0000 max=33096 max_rank=2 min=32241 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64974 max_load_reduction_pct=0.662527 moved_to_replicas=195044 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[32185, 32765, 32774, 32775, 33096, 32764, 32689, 33096] avg=32768.0000 max=33096 max_rank=4 min=32185 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30524 max_load_reduction_pct=0.479786 moved_to_replicas=64278 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[32185, 32765, 32774, 32775, 33096, 32764, 32689, 33096] avg=32768.0000 max=33096 max_rank=4 min=32185 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30524 max_load_reduction_pct=0.479786 moved_to_replicas=64278 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[36, 32765, 32774, 32775, 63620, 32764, 32689, 34721] avg=32768.0000 max=63620 max_rank=4 min=36 min_rank=0 imbalance_ratio=1.941528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[32241, 32569, 33096, 32760, 33096, 32778, 32765, 32839] avg=32768.0000 max=33096 max_rank=2 min=32241 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64974 max_load_reduction_pct=0.662527 moved_to_replicas=195044 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[33096, 33092, 33096, 32776, 31953, 32641, 32767, 32723] avg=32768.0000 max=33096 max_rank=0 min=31953 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64659 max_load_reduction_pct=0.661439 moved_to_replicas=194394 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[16, 89, 65576, 11, 98070, 32778, 32765, 32839] avg=32768.0000 max=98070 max_rank=4 min=11 min_rank=3 imbalance_ratio=2.992859 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[32241, 32569, 33096, 32760, 33096, 32778, 32765, 32839] avg=32768.0000 max=33096 max_rank=2 min=32241 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64974 max_load_reduction_pct=0.662527 moved_to_replicas=195044 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[36, 32765, 32774, 32775, 63620, 32764, 32689, 34721] avg=32768.0000 max=63620 max_rank=4 min=36 min_rank=0 imbalance_ratio=1.941528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[32185, 32765, 32774, 32775, 33096, 32764, 32689, 33096] avg=32768.0000 max=33096 max_rank=4 min=32185 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30524 max_load_reduction_pct=0.479786 moved_to_replicas=64278 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[97755, 692, 65496, 32776, 17, 32641, 32767, 0] avg=32768.0000 max=97755 max_rank=0 min=0 min_rank=7 imbalance_ratio=2.983246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[97755, 692, 65496, 32776, 17, 32641, 32767, 0] avg=32768.0000 max=97755 max_rank=0 min=0 min_rank=7 imbalance_ratio=2.983246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[32241, 32569, 33096, 32760, 33096, 32778, 32765, 32839] avg=32768.0000 max=33096 max_rank=2 min=32241 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64974 max_load_reduction_pct=0.662527 moved_to_replicas=195044 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[36, 32765, 32774, 32775, 63620, 32764, 32689, 34721] avg=32768.0000 max=63620 max_rank=4 min=36 min_rank=0 imbalance_ratio=1.941528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[32185, 32765, 32774, 32775, 33096, 32764, 32689, 33096] avg=32768.0000 max=33096 max_rank=4 min=32185 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30524 max_load_reduction_pct=0.479786 moved_to_replicas=64278 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[33096, 33092, 33096, 32776, 31953, 32641, 32767, 32723] avg=32768.0000 max=33096 max_rank=0 min=31953 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64659 max_load_reduction_pct=0.661439 moved_to_replicas=194394 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[36, 32765, 32774, 32775, 63620, 32764, 32689, 34721] avg=32768.0000 max=63620 max_rank=4 min=36 min_rank=0 imbalance_ratio=1.941528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[33096, 33092, 33096, 32776, 31953, 32641, 32767, 32723] avg=32768.0000 max=33096 max_rank=0 min=31953 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64659 max_load_reduction_pct=0.661439 moved_to_replicas=194394 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[97755, 692, 65496, 32776, 17, 32641, 32767, 0] avg=32768.0000 max=97755 max_rank=0 min=0 min_rank=7 imbalance_ratio=2.983246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[32185, 32765, 32774, 32775, 33096, 32764, 32689, 33096] avg=32768.0000 max=33096 max_rank=4 min=32185 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30524 max_load_reduction_pct=0.479786 moved_to_replicas=64278 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[36, 32765, 32774, 32775, 63620, 32764, 32689, 34721] avg=32768.0000 max=63620 max_rank=4 min=36 min_rank=0 imbalance_ratio=1.941528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[32185, 32765, 32774, 32775, 33096, 32764, 32689, 33096] avg=32768.0000 max=33096 max_rank=4 min=32185 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30524 max_load_reduction_pct=0.479786 moved_to_replicas=64278 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[97755, 692, 65496, 32776, 17, 32641, 32767, 0] avg=32768.0000 max=97755 max_rank=0 min=0 min_rank=7 imbalance_ratio=2.983246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[33096, 33092, 33096, 32776, 31953, 32641, 32767, 32723] avg=32768.0000 max=33096 max_rank=0 min=31953 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64659 max_load_reduction_pct=0.661439 moved_to_replicas=194394 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[32185, 32765, 32774, 32775, 33096, 32764, 32689, 33096] avg=32768.0000 max=33096 max_rank=4 min=32185 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30524 max_load_reduction_pct=0.479786 moved_to_replicas=64278 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[97755, 692, 65496, 32776, 17, 32641, 32767, 0] avg=32768.0000 max=97755 max_rank=0 min=0 min_rank=7 imbalance_ratio=2.983246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[33096, 33092, 33096, 32776, 31953, 32641, 32767, 32723] avg=32768.0000 max=33096 max_rank=0 min=31953 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64659 max_load_reduction_pct=0.661439 moved_to_replicas=194394 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[97755, 692, 65496, 32776, 17, 32641, 32767, 0] avg=32768.0000 max=97755 max_rank=0 min=0 min_rank=7 imbalance_ratio=2.983246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[33096, 33092, 33096, 32776, 31953, 32641, 32767, 32723] avg=32768.0000 max=33096 max_rank=0 min=31953 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64659 max_load_reduction_pct=0.661439 moved_to_replicas=194394 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[97755, 692, 65496, 32776, 17, 32641, 32767, 0] avg=32768.0000 max=97755 max_rank=0 min=0 min_rank=7 imbalance_ratio=2.983246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[33096, 33092, 33096, 32776, 31953, 32641, 32767, 32723] avg=32768.0000 max=33096 max_rank=0 min=31953 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64659 max_load_reduction_pct=0.661439 moved_to_replicas=194394 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[33096, 33092, 33096, 32776, 31953, 32641, 32767, 32723] avg=32768.0000 max=33096 max_rank=0 min=31953 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64659 max_load_reduction_pct=0.661439 moved_to_replicas=194394 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=32 total_expected_recv=1048907 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57600 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=32 total_expected_recv=1047270 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=55040 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=32 total_expected_recv=1047767 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56064 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=32 total_expected_recv=1052989 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=50816 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=32 total_expected_recv=1049314 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=55040 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=32 total_expected_recv=1048708 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57216 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=32 total_expected_recv=1046995 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=58112 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=32 total_expected_recv=1046658 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57984 peak_demand=33098 required_capacity=1.010071 + [2026-08-30 05:11:45] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 554.9 | throughput per GPU (TFLOP/s/GPU): 600.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.278322E+00 | loss scale: 1.0 | grad norm: 5.719 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=34816 expected_recv=64376 expected_overflow=0 exact_recv_per_rank=[31790, 33095, 33096, 33097, 33096, 31777, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31790, 33095, 33096, 33097, 33096, 31777, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=34816 expected_recv=64872 expected_overflow=0 exact_recv_per_rank=[31790, 33095, 33096, 33097, 33096, 31777, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31790, 33095, 33096, 33097, 33096, 31777, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=34816 expected_recv=64077 expected_overflow=0 exact_recv_per_rank=[31790, 33095, 33096, 33097, 33096, 31777, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31790, 33095, 33096, 33097, 33096, 31777, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=34816 expected_recv=64376 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31790, 33095, 33096, 33097, 33096, 31777, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31790, 33095, 33096, 33097, 33096, 31777, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=34816 expected_recv=64872 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=64896 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=34816 expected_recv=64077 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=64768 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=34816 expected_recv=65525 expected_overflow=0 exact_recv_per_rank=[31753, 32759, 33096, 33096, 32769, 32770, 33096, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31753, 32759, 33096, 33096, 32769, 32770, 33096, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=34816 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[31753, 32759, 33096, 33096, 32769, 32770, 33096, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[31753, 32759, 33096, 33096, 32769, 32770, 33096, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=34816 expected_recv=63513 expected_overflow=0 exact_recv_per_rank=[31753, 32759, 33096, 33096, 32769, 32770, 33096, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=34816 expected_recv=65609 expected_overflow=0 exact_recv_per_rank=[31753, 32759, 33096, 33096, 32769, 32770, 33096, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=34816 expected_recv=65525 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=34816 expected_recv=65531 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=34816 expected_recv=63513 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31753, 32759, 33096, 33096, 32769, 32770, 33096, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=34816 expected_recv=65609 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=34816 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[31753, 32759, 33096, 33096, 32769, 32770, 33096, 32805] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=67328 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=64384 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=34816 expected_recv=65534 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32788, 32766, 33096, 32774, 32436, 33096, 32410, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32788, 32766, 33096, 32774, 32436, 33096, 32410, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=34816 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[32788, 32766, 33096, 32774, 32436, 33096, 32410, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=34816 expected_recv=64835 expected_overflow=0 exact_recv_per_rank=[32788, 32766, 33096, 32774, 32436, 33096, 32410, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=34816 expected_recv=65541 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=34816 expected_recv=65581 expected_overflow=0 exact_recv_per_rank=[32788, 32766, 33096, 32774, 32436, 33096, 32410, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=34816 expected_recv=64835 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=34816 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[32788, 32766, 33096, 32774, 32436, 33096, 32410, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=34816 expected_recv=65581 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=34816 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[32788, 32766, 33096, 32774, 32436, 33096, 32410, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=34816 expected_recv=64860 expected_overflow=0 exact_recv_per_rank=[32788, 32766, 33096, 32774, 32436, 33096, 32410, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=34816 expected_recv=65546 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=34816 expected_recv=65541 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=34816 expected_recv=64860 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32176, 32398, 33095, 32414, 33096, 33097, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=34816 expected_recv=65330 expected_overflow=0 exact_recv_per_rank=[32176, 32398, 33095, 32414, 33096, 33097, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=34816 expected_recv=64382 expected_overflow=0 exact_recv_per_rank=[32176, 32398, 33095, 32414, 33096, 33097, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=34816 expected_recv=65330 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32176, 32398, 33095, 32414, 33096, 33097, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=34816 expected_recv=64382 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=34816 expected_recv=65272 expected_overflow=0 exact_recv_per_rank=[32176, 32398, 33095, 32414, 33096, 33097, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=34816 expected_recv=65549 expected_overflow=0 exact_recv_per_rank=[32176, 32398, 33095, 32414, 33096, 33097, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=34816 expected_recv=65272 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=34816 expected_recv=65549 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32176, 32398, 33095, 32414, 33096, 33097, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=34816 expected_recv=65180 expected_overflow=0 exact_recv_per_rank=[32176, 32398, 33095, 32414, 33096, 33097, 33096, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=34816 expected_recv=65180 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=34816 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[32343, 33096, 33098, 32783, 32766, 33096, 32771, 32191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32343, 33096, 33098, 32783, 32766, 33096, 32771, 32191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=34816 expected_recv=66194 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32343, 33096, 33098, 32783, 32766, 33096, 32771, 32191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=34816 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[32343, 33096, 33098, 32783, 32766, 33096, 32771, 32191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=34816 expected_recv=65114 expected_overflow=0 exact_recv_per_rank=[32343, 33096, 33098, 32783, 32766, 33096, 32771, 32191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=34816 expected_recv=64339 expected_overflow=0 exact_recv_per_rank=[32343, 33096, 33098, 32783, 32766, 33096, 32771, 32191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=34816 expected_recv=65545 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=34816 expected_recv=65160 expected_overflow=0 exact_recv_per_rank=[32343, 33096, 33098, 32783, 32766, 33096, 32771, 32191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=34816 expected_recv=65114 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=34816 expected_recv=64339 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=34816 expected_recv=65552 expected_overflow=0 exact_recv_per_rank=[32343, 33096, 33098, 32783, 32766, 33096, 32771, 32191] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=34816 expected_recv=65160 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=34816 expected_recv=65552 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=34816 expected_recv=65617 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32521, 32758, 33096, 32773, 33096, 31708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=34816 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32521, 32758, 33096, 32773, 33096, 31708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32521, 32758, 33096, 32773, 33096, 31708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=34816 expected_recv=65617 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=34816 expected_recv=65546 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32521, 32758, 33096, 32773, 33096, 31708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32521, 32758, 33096, 32773, 33096, 31708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=34816 expected_recv=63594 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32521, 32758, 33096, 32773, 33096, 31708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=34816 expected_recv=65459 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32521, 32758, 33096, 32773, 33096, 31708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=34816 expected_recv=65496 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32521, 32758, 33096, 32773, 33096, 31708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=34816 expected_recv=63594 actual_valid_recv=64128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=34816 expected_recv=65459 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=34816 expected_recv=65496 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=64128 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=34816 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[32800, 33096, 31872, 33096, 33096, 32764, 32324, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=34816 expected_recv=63716 expected_overflow=0 exact_recv_per_rank=[32800, 33096, 31872, 33096, 33096, 32764, 32324, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32800, 33096, 31872, 33096, 33096, 32764, 32324, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=34816 expected_recv=65529 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=34816 expected_recv=63716 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=34816 expected_recv=64696 expected_overflow=0 exact_recv_per_rank=[32800, 33096, 31872, 33096, 33096, 32764, 32324, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=34816 expected_recv=65579 expected_overflow=0 exact_recv_per_rank=[32800, 33096, 31872, 33096, 33096, 32764, 32324, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=64640 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32800, 33096, 31872, 33096, 33096, 32764, 32324, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=34816 expected_recv=64696 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32800, 33096, 31872, 33096, 33096, 32764, 32324, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=34816 expected_recv=65579 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32800, 33096, 31872, 33096, 33096, 32764, 32324, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=34816 expected_recv=65477 expected_overflow=0 exact_recv_per_rank=[33036, 32088, 33096, 32244, 33096, 32726, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33036, 32088, 33096, 32244, 33096, 32726, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=34816 expected_recv=65477 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=34816 expected_recv=64272 expected_overflow=0 exact_recv_per_rank=[33036, 32088, 33096, 32244, 33096, 32726, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=34816 expected_recv=64272 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33036, 32088, 33096, 32244, 33096, 32726, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=34816 expected_recv=65943 expected_overflow=0 exact_recv_per_rank=[33036, 32088, 33096, 32244, 33096, 32726, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33036, 32088, 33096, 32244, 33096, 32726, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=34816 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[33036, 32088, 33096, 32244, 33096, 32726, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=34816 expected_recv=64488 expected_overflow=0 exact_recv_per_rank=[33036, 32088, 33096, 32244, 33096, 32726, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=34816 expected_recv=65943 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=34816 expected_recv=65532 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=34816 expected_recv=64488 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=34816 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[33096, 32771, 33096, 32774, 31966, 32773, 32572, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32771, 33096, 32774, 31966, 32773, 32572, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65024 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=34816 expected_recv=65543 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=34816 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[33096, 32771, 33096, 32774, 31966, 32773, 32572, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=34816 expected_recv=65536 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=34816 expected_recv=65054 expected_overflow=0 exact_recv_per_rank=[33096, 32771, 33096, 32774, 31966, 32773, 32572, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=34816 expected_recv=64032 expected_overflow=0 exact_recv_per_rank=[33096, 32771, 33096, 32774, 31966, 32773, 32572, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 32771, 33096, 32774, 31966, 32773, 32572, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 32771, 33096, 32774, 31966, 32773, 32572, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=34816 expected_recv=65054 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=34816 expected_recv=64032 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=34816 expected_recv=65547 expected_overflow=0 exact_recv_per_rank=[33096, 32771, 33096, 32774, 31966, 32773, 32572, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=64640 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=34816 expected_recv=65547 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32294, 32765, 33096, 31605, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32294, 32765, 33096, 31605, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=34816 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[32294, 32765, 33096, 31605, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=67200 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=34816 expected_recv=65530 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=34816 expected_recv=65370 expected_overflow=0 exact_recv_per_rank=[32294, 32765, 33096, 31605, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32294, 32765, 33096, 31605, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=34816 expected_recv=64659 expected_overflow=0 exact_recv_per_rank=[32294, 32765, 33096, 31605, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=34816 expected_recv=65370 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=34816 expected_recv=64659 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32294, 32765, 33096, 31605, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=34816 expected_recv=63962 expected_overflow=0 exact_recv_per_rank=[32294, 32765, 33096, 31605, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=34816 expected_recv=65079 expected_overflow=0 exact_recv_per_rank=[33096, 32440, 32448, 32710, 33096, 32500, 33096, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=34816 expected_recv=63962 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=34816 expected_recv=64893 expected_overflow=0 exact_recv_per_rank=[33096, 32440, 32448, 32710, 33096, 32500, 33096, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=34816 expected_recv=65079 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65024 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=34816 expected_recv=64883 expected_overflow=0 exact_recv_per_rank=[33096, 32440, 32448, 32710, 33096, 32500, 33096, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=34816 expected_recv=64893 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=34816 expected_recv=64883 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32440, 32448, 32710, 33096, 32500, 33096, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32440, 32448, 32710, 33096, 32500, 33096, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32440, 32448, 32710, 33096, 32500, 33096, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=34816 expected_recv=65517 expected_overflow=0 exact_recv_per_rank=[33096, 32440, 32448, 32710, 33096, 32500, 33096, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=34816 expected_recv=65340 expected_overflow=0 exact_recv_per_rank=[33096, 32440, 32448, 32710, 33096, 32500, 33096, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=34816 expected_recv=64901 expected_overflow=0 exact_recv_per_rank=[32745, 33096, 32780, 33096, 32740, 32436, 32492, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=34816 expected_recv=65517 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=34816 expected_recv=65340 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=34816 expected_recv=65560 expected_overflow=0 exact_recv_per_rank=[32745, 33096, 32780, 33096, 32740, 32436, 32492, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=34816 expected_recv=64901 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=34816 expected_recv=65560 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32745, 33096, 32780, 33096, 32740, 32436, 32492, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=34816 expected_recv=65449 expected_overflow=0 exact_recv_per_rank=[32745, 33096, 32780, 33096, 32740, 32436, 32492, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=34816 expected_recv=65449 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=34816 expected_recv=65497 expected_overflow=0 exact_recv_per_rank=[32745, 33096, 32780, 33096, 32740, 32436, 32492, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=34816 expected_recv=64977 expected_overflow=0 exact_recv_per_rank=[32745, 33096, 32780, 33096, 32740, 32436, 32492, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=34816 expected_recv=65497 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=34816 expected_recv=64977 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=34816 expected_recv=65520 expected_overflow=0 exact_recv_per_rank=[32745, 33096, 32780, 33096, 32740, 32436, 32492, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32745, 33096, 32780, 33096, 32740, 32436, 32492, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=34816 expected_recv=65116 expected_overflow=0 exact_recv_per_rank=[32476, 32795, 33096, 32742, 32757, 32494, 33026, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=34816 expected_recv=65520 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32476, 32795, 33096, 32742, 32757, 32494, 33026, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=34816 expected_recv=65116 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=34816 expected_recv=65580 expected_overflow=0 exact_recv_per_rank=[32476, 32795, 33096, 32742, 32757, 32494, 33026, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=34816 expected_recv=65519 expected_overflow=0 exact_recv_per_rank=[32476, 32795, 33096, 32742, 32757, 32494, 33026, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=34816 expected_recv=65580 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=34816 expected_recv=65519 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=34816 expected_recv=64932 expected_overflow=0 exact_recv_per_rank=[32476, 32795, 33096, 32742, 32757, 32494, 33026, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=34816 expected_recv=65945 expected_overflow=0 exact_recv_per_rank=[32476, 32795, 33096, 32742, 32757, 32494, 33026, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=34816 expected_recv=64932 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=34816 expected_recv=65945 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=34816 expected_recv=65510 expected_overflow=0 exact_recv_per_rank=[32476, 32795, 33096, 32742, 32757, 32494, 33026, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=34816 expected_recv=65494 expected_overflow=0 exact_recv_per_rank=[32476, 32795, 33096, 32742, 32757, 32494, 33026, 32758] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=34816 expected_recv=65510 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=34816 expected_recv=65552 expected_overflow=0 exact_recv_per_rank=[31942, 32908, 33096, 32764, 33096, 32782, 32765, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31942, 32908, 33096, 32764, 33096, 32782, 32765, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=34816 expected_recv=65494 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=34816 expected_recv=65552 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=34816 expected_recv=65749 expected_overflow=0 exact_recv_per_rank=[31942, 32908, 33096, 32764, 33096, 32782, 32765, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31942, 32908, 33096, 32764, 33096, 32782, 32765, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=34816 expected_recv=65749 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=34816 expected_recv=63959 expected_overflow=0 exact_recv_per_rank=[31942, 32908, 33096, 32764, 33096, 32782, 32765, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=34816 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[31942, 32908, 33096, 32764, 33096, 32782, 32765, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=34816 expected_recv=63959 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=34816 expected_recv=65532 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=64896 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=34816 expected_recv=65582 expected_overflow=0 exact_recv_per_rank=[31942, 32908, 33096, 32764, 33096, 32782, 32765, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=34816 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[31942, 32908, 33096, 32764, 33096, 32782, 32765, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=34816 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[32762, 33023, 32053, 32771, 33096, 32777, 32566, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=34816 expected_recv=65582 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=34816 expected_recv=64211 expected_overflow=0 exact_recv_per_rank=[32762, 33023, 32053, 32771, 33096, 32777, 32566, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=34816 expected_recv=65530 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=34816 expected_recv=66043 expected_overflow=0 exact_recv_per_rank=[32762, 33023, 32053, 32771, 33096, 32777, 32566, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=34816 expected_recv=65545 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32762, 33023, 32053, 32771, 33096, 32777, 32566, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=34816 expected_recv=64211 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=34816 expected_recv=66043 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=34816 expected_recv=65356 expected_overflow=0 exact_recv_per_rank=[32762, 33023, 32053, 32771, 33096, 32777, 32566, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=34816 expected_recv=65209 expected_overflow=0 exact_recv_per_rank=[32762, 33023, 32053, 32771, 33096, 32777, 32566, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=34816 expected_recv=65356 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=34816 expected_recv=65209 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32762, 33023, 32053, 32771, 33096, 32777, 32566, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=34816 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[32762, 33023, 32053, 32771, 33096, 32777, 32566, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32456, 33096, 32766, 32167, 33096, 32768, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32456, 33096, 32766, 32167, 33096, 32768, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=34816 expected_recv=65540 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=34816 expected_recv=64665 expected_overflow=0 exact_recv_per_rank=[33096, 32456, 33096, 32766, 32167, 33096, 32768, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=34816 expected_recv=64549 expected_overflow=0 exact_recv_per_rank=[33096, 32456, 33096, 32766, 32167, 33096, 32768, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=34816 expected_recv=64665 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=34816 expected_recv=64549 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32456, 33096, 32766, 32167, 33096, 32768, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=34816 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[33096, 32456, 33096, 32766, 32167, 33096, 32768, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=34816 expected_recv=65536 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=34816 expected_recv=65430 expected_overflow=0 exact_recv_per_rank=[33096, 32456, 33096, 32766, 32167, 33096, 32768, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=34816 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[33096, 32456, 33096, 32766, 32167, 33096, 32768, 32699] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=34816 expected_recv=65430 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[31790, 26965, 16464, 25611, 7452, 27357, 49634, 76871] avg=32768.0000 max=76871 max_rank=7 min=7452 min_rank=4 imbalance_ratio=2.345917 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=34816 expected_recv=65532 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[31790, 26965, 16464, 25611, 7452, 27357, 49634, 76871] avg=32768.0000 max=76871 max_rank=7 min=7452 min_rank=4 imbalance_ratio=2.345917 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[31790, 26965, 16464, 25611, 7452, 27357, 49634, 76871] avg=32768.0000 max=76871 max_rank=7 min=7452 min_rank=4 imbalance_ratio=2.345917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[31790, 26965, 16464, 25611, 7452, 27357, 49634, 76871] avg=32768.0000 max=76871 max_rank=7 min=7452 min_rank=4 imbalance_ratio=2.345917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[31790, 33095, 33096, 33097, 33096, 31777, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31777 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=43774 max_load_reduction_pct=0.569448 moved_to_replicas=118845 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[31790, 26965, 16464, 25611, 7452, 27357, 49634, 76871] avg=32768.0000 max=76871 max_rank=7 min=7452 min_rank=4 imbalance_ratio=2.345917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[31790, 26965, 16464, 25611, 7452, 27357, 49634, 76871] avg=32768.0000 max=76871 max_rank=7 min=7452 min_rank=4 imbalance_ratio=2.345917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[31790, 33095, 33096, 33097, 33096, 31777, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31777 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=43774 max_load_reduction_pct=0.569448 moved_to_replicas=118845 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[31790, 33095, 33096, 33097, 33096, 31777, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31777 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=43774 max_load_reduction_pct=0.569448 moved_to_replicas=118845 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[31790, 33095, 33096, 33097, 33096, 31777, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31777 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=43774 max_load_reduction_pct=0.569448 moved_to_replicas=118845 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[31790, 26965, 16464, 25611, 7452, 27357, 49634, 76871] avg=32768.0000 max=76871 max_rank=7 min=7452 min_rank=4 imbalance_ratio=2.345917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[31790, 26965, 16464, 25611, 7452, 27357, 49634, 76871] avg=32768.0000 max=76871 max_rank=7 min=7452 min_rank=4 imbalance_ratio=2.345917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[31790, 33095, 33096, 33097, 33096, 31777, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31777 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=43774 max_load_reduction_pct=0.569448 moved_to_replicas=118845 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[31790, 33095, 33096, 33097, 33096, 31777, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31777 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=43774 max_load_reduction_pct=0.569448 moved_to_replicas=118845 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[28366, 32759, 79292, 50700, 32769, 11, 5442, 32805] avg=32768.0000 max=79292 max_rank=2 min=11 min_rank=5 imbalance_ratio=2.419800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[31790, 33095, 33096, 33097, 33096, 31777, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31777 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=43774 max_load_reduction_pct=0.569448 moved_to_replicas=118845 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[28366, 32759, 79292, 50700, 32769, 11, 5442, 32805] avg=32768.0000 max=79292 max_rank=2 min=11 min_rank=5 imbalance_ratio=2.419800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[31790, 33095, 33096, 33097, 33096, 31777, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31777 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=43774 max_load_reduction_pct=0.569448 moved_to_replicas=118845 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[28366, 32759, 79292, 50700, 32769, 11, 5442, 32805] avg=32768.0000 max=79292 max_rank=2 min=11 min_rank=5 imbalance_ratio=2.419800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[28366, 32759, 79292, 50700, 32769, 11, 5442, 32805] avg=32768.0000 max=79292 max_rank=2 min=11 min_rank=5 imbalance_ratio=2.419800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[31753, 32759, 33096, 33096, 32769, 32770, 33096, 32805] avg=32768.0000 max=33096 max_rank=2 min=31753 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46196 max_load_reduction_pct=0.582606 moved_to_replicas=131194 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[28366, 32759, 79292, 50700, 32769, 11, 5442, 32805] avg=32768.0000 max=79292 max_rank=2 min=11 min_rank=5 imbalance_ratio=2.419800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[28366, 32759, 79292, 50700, 32769, 11, 5442, 32805] avg=32768.0000 max=79292 max_rank=2 min=11 min_rank=5 imbalance_ratio=2.419800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[31753, 32759, 33096, 33096, 32769, 32770, 33096, 32805] avg=32768.0000 max=33096 max_rank=2 min=31753 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46196 max_load_reduction_pct=0.582606 moved_to_replicas=131194 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[31753, 32759, 33096, 33096, 32769, 32770, 33096, 32805] avg=32768.0000 max=33096 max_rank=2 min=31753 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46196 max_load_reduction_pct=0.582606 moved_to_replicas=131194 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[31753, 32759, 33096, 33096, 32769, 32770, 33096, 32805] avg=32768.0000 max=33096 max_rank=2 min=31753 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46196 max_load_reduction_pct=0.582606 moved_to_replicas=131194 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[28366, 32759, 79292, 50700, 32769, 11, 5442, 32805] avg=32768.0000 max=79292 max_rank=2 min=11 min_rank=5 imbalance_ratio=2.419800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[31753, 32759, 33096, 33096, 32769, 32770, 33096, 32805] avg=32768.0000 max=33096 max_rank=2 min=31753 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46196 max_load_reduction_pct=0.582606 moved_to_replicas=131194 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[28366, 32759, 79292, 50700, 32769, 11, 5442, 32805] avg=32768.0000 max=79292 max_rank=2 min=11 min_rank=5 imbalance_ratio=2.419800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[32788, 32766, 63976, 32774, 16, 65516, 1530, 32778] avg=32768.0000 max=65516 max_rank=5 min=16 min_rank=4 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[31753, 32759, 33096, 33096, 32769, 32770, 33096, 32805] avg=32768.0000 max=33096 max_rank=2 min=31753 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46196 max_load_reduction_pct=0.582606 moved_to_replicas=131194 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[31753, 32759, 33096, 33096, 32769, 32770, 33096, 32805] avg=32768.0000 max=33096 max_rank=2 min=31753 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46196 max_load_reduction_pct=0.582606 moved_to_replicas=131194 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[32788, 32766, 63976, 32774, 16, 65516, 1530, 32778] avg=32768.0000 max=65516 max_rank=5 min=16 min_rank=4 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[32788, 32766, 63976, 32774, 16, 65516, 1530, 32778] avg=32768.0000 max=65516 max_rank=5 min=16 min_rank=4 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[32788, 32766, 63976, 32774, 16, 65516, 1530, 32778] avg=32768.0000 max=65516 max_rank=5 min=16 min_rank=4 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[31753, 32759, 33096, 33096, 32769, 32770, 33096, 32805] avg=32768.0000 max=33096 max_rank=2 min=31753 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46196 max_load_reduction_pct=0.582606 moved_to_replicas=131194 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[32788, 32766, 33096, 32774, 32436, 33096, 32410, 32778] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=126036 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[32788, 32766, 63976, 32774, 16, 65516, 1530, 32778] avg=32768.0000 max=65516 max_rank=5 min=16 min_rank=4 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[32788, 32766, 63976, 32774, 16, 65516, 1530, 32778] avg=32768.0000 max=65516 max_rank=5 min=16 min_rank=4 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[32788, 32766, 33096, 32774, 32436, 33096, 32410, 32778] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=126036 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[32788, 32766, 33096, 32774, 32436, 33096, 32410, 32778] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=126036 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[32788, 32766, 33096, 32774, 32436, 33096, 32410, 32778] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=126036 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[32788, 32766, 63976, 32774, 16, 65516, 1530, 32778] avg=32768.0000 max=65516 max_rank=5 min=16 min_rank=4 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[32788, 32766, 33096, 32774, 32436, 33096, 32410, 32778] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=126036 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[15, 31958, 51776, 11, 65499, 14856, 65257, 32772] avg=32768.0000 max=65499 max_rank=4 min=11 min_rank=3 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[32788, 32766, 63976, 32774, 16, 65516, 1530, 32778] avg=32768.0000 max=65516 max_rank=5 min=16 min_rank=4 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[32788, 32766, 33096, 32774, 32436, 33096, 32410, 32778] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=126036 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[15, 31958, 51776, 11, 65499, 14856, 65257, 32772] avg=32768.0000 max=65499 max_rank=4 min=11 min_rank=3 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[32788, 32766, 33096, 32774, 32436, 33096, 32410, 32778] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=126036 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[15, 31958, 51776, 11, 65499, 14856, 65257, 32772] avg=32768.0000 max=65499 max_rank=4 min=11 min_rank=3 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[15, 31958, 51776, 11, 65499, 14856, 65257, 32772] avg=32768.0000 max=65499 max_rank=4 min=11 min_rank=3 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[32176, 32398, 33095, 32414, 33096, 33097, 33096, 32772] avg=32768.0000 max=33097 max_rank=5 min=32176 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32402 max_load_reduction_pct=0.494695 moved_to_replicas=165745 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[32788, 32766, 33096, 32774, 32436, 33096, 32410, 32778] avg=32768.0000 max=33096 max_rank=2 min=32410 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=126036 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[15, 31958, 51776, 11, 65499, 14856, 65257, 32772] avg=32768.0000 max=65499 max_rank=4 min=11 min_rank=3 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[32176, 32398, 33095, 32414, 33096, 33097, 33096, 32772] avg=32768.0000 max=33097 max_rank=5 min=32176 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32402 max_load_reduction_pct=0.494695 moved_to_replicas=165745 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[15, 31958, 51776, 11, 65499, 14856, 65257, 32772] avg=32768.0000 max=65499 max_rank=4 min=11 min_rank=3 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[32176, 32398, 33095, 32414, 33096, 33097, 33096, 32772] avg=32768.0000 max=33097 max_rank=5 min=32176 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32402 max_load_reduction_pct=0.494695 moved_to_replicas=165745 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[32176, 32398, 33095, 32414, 33096, 33097, 33096, 32772] avg=32768.0000 max=33097 max_rank=5 min=32176 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32402 max_load_reduction_pct=0.494695 moved_to_replicas=165745 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[15, 31958, 51776, 11, 65499, 14856, 65257, 32772] avg=32768.0000 max=65499 max_rank=4 min=11 min_rank=3 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[24, 98176, 38410, 32783, 5, 34125, 32771, 25850] avg=32768.0000 max=98176 max_rank=1 min=5 min_rank=4 imbalance_ratio=2.996094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[32176, 32398, 33095, 32414, 33096, 33097, 33096, 32772] avg=32768.0000 max=33097 max_rank=5 min=32176 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32402 max_load_reduction_pct=0.494695 moved_to_replicas=165745 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[32176, 32398, 33095, 32414, 33096, 33097, 33096, 32772] avg=32768.0000 max=33097 max_rank=5 min=32176 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32402 max_load_reduction_pct=0.494695 moved_to_replicas=165745 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[15, 31958, 51776, 11, 65499, 14856, 65257, 32772] avg=32768.0000 max=65499 max_rank=4 min=11 min_rank=3 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[24, 98176, 38410, 32783, 5, 34125, 32771, 25850] avg=32768.0000 max=98176 max_rank=1 min=5 min_rank=4 imbalance_ratio=2.996094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[32176, 32398, 33095, 32414, 33096, 33097, 33096, 32772] avg=32768.0000 max=33097 max_rank=5 min=32176 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32402 max_load_reduction_pct=0.494695 moved_to_replicas=165745 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[32343, 33096, 33098, 32783, 32766, 33096, 32771, 32191] avg=32768.0000 max=33098 max_rank=2 min=32191 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=65078 max_load_reduction_pct=0.662871 moved_to_replicas=142030 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[24, 98176, 38410, 32783, 5, 34125, 32771, 25850] avg=32768.0000 max=98176 max_rank=1 min=5 min_rank=4 imbalance_ratio=2.996094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[24, 98176, 38410, 32783, 5, 34125, 32771, 25850] avg=32768.0000 max=98176 max_rank=1 min=5 min_rank=4 imbalance_ratio=2.996094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[32176, 32398, 33095, 32414, 33096, 33097, 33096, 32772] avg=32768.0000 max=33097 max_rank=5 min=32176 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32402 max_load_reduction_pct=0.494695 moved_to_replicas=165745 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[32343, 33096, 33098, 32783, 32766, 33096, 32771, 32191] avg=32768.0000 max=33098 max_rank=2 min=32191 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=65078 max_load_reduction_pct=0.662871 moved_to_replicas=142030 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[24, 98176, 38410, 32783, 5, 34125, 32771, 25850] avg=32768.0000 max=98176 max_rank=1 min=5 min_rank=4 imbalance_ratio=2.996094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[24, 98176, 38410, 32783, 5, 34125, 32771, 25850] avg=32768.0000 max=98176 max_rank=1 min=5 min_rank=4 imbalance_ratio=2.996094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[32343, 33096, 33098, 32783, 32766, 33096, 32771, 32191] avg=32768.0000 max=33098 max_rank=2 min=32191 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=65078 max_load_reduction_pct=0.662871 moved_to_replicas=142030 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[32343, 33096, 33098, 32783, 32766, 33096, 32771, 32191] avg=32768.0000 max=33098 max_rank=2 min=32191 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=65078 max_load_reduction_pct=0.662871 moved_to_replicas=142030 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[24, 98176, 38410, 32783, 5, 34125, 32771, 25850] avg=32768.0000 max=98176 max_rank=1 min=5 min_rank=4 imbalance_ratio=2.996094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[37086, 64143, 78, 32758, 30502, 32773, 64798, 6] avg=32768.0000 max=64798 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.977478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[32343, 33096, 33098, 32783, 32766, 33096, 32771, 32191] avg=32768.0000 max=33098 max_rank=2 min=32191 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=65078 max_load_reduction_pct=0.662871 moved_to_replicas=142030 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[32343, 33096, 33098, 32783, 32766, 33096, 32771, 32191] avg=32768.0000 max=33098 max_rank=2 min=32191 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=65078 max_load_reduction_pct=0.662871 moved_to_replicas=142030 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[37086, 64143, 78, 32758, 30502, 32773, 64798, 6] avg=32768.0000 max=64798 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.977478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[24, 98176, 38410, 32783, 5, 34125, 32771, 25850] avg=32768.0000 max=98176 max_rank=1 min=5 min_rank=4 imbalance_ratio=2.996094 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[32343, 33096, 33098, 32783, 32766, 33096, 32771, 32191] avg=32768.0000 max=33098 max_rank=2 min=32191 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=65078 max_load_reduction_pct=0.662871 moved_to_replicas=142030 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 32521, 32758, 33096, 32773, 33096, 31708] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31702 max_load_reduction_pct=0.489243 moved_to_replicas=133253 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[37086, 64143, 78, 32758, 30502, 32773, 64798, 6] avg=32768.0000 max=64798 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.977478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[37086, 64143, 78, 32758, 30502, 32773, 64798, 6] avg=32768.0000 max=64798 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.977478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 32521, 32758, 33096, 32773, 33096, 31708] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31702 max_load_reduction_pct=0.489243 moved_to_replicas=133253 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[32343, 33096, 33098, 32783, 32766, 33096, 32771, 32191] avg=32768.0000 max=33098 max_rank=2 min=32191 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=65078 max_load_reduction_pct=0.662871 moved_to_replicas=142030 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[37086, 64143, 78, 32758, 30502, 32773, 64798, 6] avg=32768.0000 max=64798 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.977478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[37086, 64143, 78, 32758, 30502, 32773, 64798, 6] avg=32768.0000 max=64798 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.977478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 32521, 32758, 33096, 32773, 33096, 31708] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31702 max_load_reduction_pct=0.489243 moved_to_replicas=133253 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 32521, 32758, 33096, 32773, 33096, 31708] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31702 max_load_reduction_pct=0.489243 moved_to_replicas=133253 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[37086, 64143, 78, 32758, 30502, 32773, 64798, 6] avg=32768.0000 max=64798 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.977478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[32800, 22901, 31516, 50311, 65409, 32764, 11, 26432] avg=32768.0000 max=65409 max_rank=4 min=11 min_rank=6 imbalance_ratio=1.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 32521, 32758, 33096, 32773, 33096, 31708] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31702 max_load_reduction_pct=0.489243 moved_to_replicas=133253 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[32800, 22901, 31516, 50311, 65409, 32764, 11, 26432] avg=32768.0000 max=65409 max_rank=4 min=11 min_rank=6 imbalance_ratio=1.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 32521, 32758, 33096, 32773, 33096, 31708] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31702 max_load_reduction_pct=0.489243 moved_to_replicas=133253 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[37086, 64143, 78, 32758, 30502, 32773, 64798, 6] avg=32768.0000 max=64798 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.977478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[32800, 33096, 31872, 33096, 33096, 32764, 32324, 33096] avg=32768.0000 max=33096 max_rank=1 min=31872 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32313 max_load_reduction_pct=0.494015 moved_to_replicas=102223 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 32521, 32758, 33096, 32773, 33096, 31708] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31702 max_load_reduction_pct=0.489243 moved_to_replicas=133253 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[32800, 22901, 31516, 50311, 65409, 32764, 11, 26432] avg=32768.0000 max=65409 max_rank=4 min=11 min_rank=6 imbalance_ratio=1.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[32800, 22901, 31516, 50311, 65409, 32764, 11, 26432] avg=32768.0000 max=65409 max_rank=4 min=11 min_rank=6 imbalance_ratio=1.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[32800, 33096, 31872, 33096, 33096, 32764, 32324, 33096] avg=32768.0000 max=33096 max_rank=1 min=31872 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32313 max_load_reduction_pct=0.494015 moved_to_replicas=102223 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[32800, 22901, 31516, 50311, 65409, 32764, 11, 26432] avg=32768.0000 max=65409 max_rank=4 min=11 min_rank=6 imbalance_ratio=1.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 32521, 32758, 33096, 32773, 33096, 31708] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31702 max_load_reduction_pct=0.489243 moved_to_replicas=133253 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[32800, 22901, 31516, 50311, 65409, 32764, 11, 26432] avg=32768.0000 max=65409 max_rank=4 min=11 min_rank=6 imbalance_ratio=1.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[32800, 33096, 31872, 33096, 33096, 32764, 32324, 33096] avg=32768.0000 max=33096 max_rank=1 min=31872 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32313 max_load_reduction_pct=0.494015 moved_to_replicas=102223 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[33036, 108, 97563, 32244, 65524, 32726, 933, 10] avg=32768.0000 max=97563 max_rank=2 min=10 min_rank=7 imbalance_ratio=2.977386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[32800, 33096, 31872, 33096, 33096, 32764, 32324, 33096] avg=32768.0000 max=33096 max_rank=1 min=31872 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32313 max_load_reduction_pct=0.494015 moved_to_replicas=102223 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[32800, 22901, 31516, 50311, 65409, 32764, 11, 26432] avg=32768.0000 max=65409 max_rank=4 min=11 min_rank=6 imbalance_ratio=1.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[32800, 33096, 31872, 33096, 33096, 32764, 32324, 33096] avg=32768.0000 max=33096 max_rank=1 min=31872 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32313 max_load_reduction_pct=0.494015 moved_to_replicas=102223 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[33036, 108, 97563, 32244, 65524, 32726, 933, 10] avg=32768.0000 max=97563 max_rank=2 min=10 min_rank=7 imbalance_ratio=2.977386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[32800, 33096, 31872, 33096, 33096, 32764, 32324, 33096] avg=32768.0000 max=33096 max_rank=1 min=31872 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32313 max_load_reduction_pct=0.494015 moved_to_replicas=102223 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[33036, 32088, 33096, 32244, 33096, 32726, 33096, 32762] avg=32768.0000 max=33096 max_rank=2 min=32088 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64467 max_load_reduction_pct=0.660773 moved_to_replicas=193504 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[32800, 22901, 31516, 50311, 65409, 32764, 11, 26432] avg=32768.0000 max=65409 max_rank=4 min=11 min_rank=6 imbalance_ratio=1.996124 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[32800, 33096, 31872, 33096, 33096, 32764, 32324, 33096] avg=32768.0000 max=33096 max_rank=1 min=31872 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32313 max_load_reduction_pct=0.494015 moved_to_replicas=102223 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[33036, 108, 97563, 32244, 65524, 32726, 933, 10] avg=32768.0000 max=97563 max_rank=2 min=10 min_rank=7 imbalance_ratio=2.977386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[33036, 32088, 33096, 32244, 33096, 32726, 33096, 32762] avg=32768.0000 max=33096 max_rank=2 min=32088 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64467 max_load_reduction_pct=0.660773 moved_to_replicas=193504 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[33036, 108, 97563, 32244, 65524, 32726, 933, 10] avg=32768.0000 max=97563 max_rank=2 min=10 min_rank=7 imbalance_ratio=2.977386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[33036, 108, 97563, 32244, 65524, 32726, 933, 10] avg=32768.0000 max=97563 max_rank=2 min=10 min_rank=7 imbalance_ratio=2.977386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[32800, 33096, 31872, 33096, 33096, 32764, 32324, 33096] avg=32768.0000 max=33096 max_rank=1 min=31872 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32313 max_load_reduction_pct=0.494015 moved_to_replicas=102223 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[33036, 108, 97563, 32244, 65524, 32726, 933, 10] avg=32768.0000 max=97563 max_rank=2 min=10 min_rank=7 imbalance_ratio=2.977386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[28700, 32771, 98285, 13, 29034, 32773, 144, 40424] avg=32768.0000 max=98285 max_rank=2 min=13 min_rank=3 imbalance_ratio=2.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[33036, 32088, 33096, 32244, 33096, 32726, 33096, 32762] avg=32768.0000 max=33096 max_rank=2 min=32088 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64467 max_load_reduction_pct=0.660773 moved_to_replicas=193504 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[33036, 108, 97563, 32244, 65524, 32726, 933, 10] avg=32768.0000 max=97563 max_rank=2 min=10 min_rank=7 imbalance_ratio=2.977386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[33036, 32088, 33096, 32244, 33096, 32726, 33096, 32762] avg=32768.0000 max=33096 max_rank=2 min=32088 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64467 max_load_reduction_pct=0.660773 moved_to_replicas=193504 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[33036, 32088, 33096, 32244, 33096, 32726, 33096, 32762] avg=32768.0000 max=33096 max_rank=2 min=32088 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64467 max_load_reduction_pct=0.660773 moved_to_replicas=193504 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[28700, 32771, 98285, 13, 29034, 32773, 144, 40424] avg=32768.0000 max=98285 max_rank=2 min=13 min_rank=3 imbalance_ratio=2.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[33036, 32088, 33096, 32244, 33096, 32726, 33096, 32762] avg=32768.0000 max=33096 max_rank=2 min=32088 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64467 max_load_reduction_pct=0.660773 moved_to_replicas=193504 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32771, 33096, 32774, 31966, 32773, 32572, 33096] avg=32768.0000 max=33096 max_rank=0 min=31966 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65189 max_load_reduction_pct=0.663265 moved_to_replicas=144548 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[33036, 108, 97563, 32244, 65524, 32726, 933, 10] avg=32768.0000 max=97563 max_rank=2 min=10 min_rank=7 imbalance_ratio=2.977386 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[33036, 32088, 33096, 32244, 33096, 32726, 33096, 32762] avg=32768.0000 max=33096 max_rank=2 min=32088 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64467 max_load_reduction_pct=0.660773 moved_to_replicas=193504 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32771, 33096, 32774, 31966, 32773, 32572, 33096] avg=32768.0000 max=33096 max_rank=0 min=31966 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65189 max_load_reduction_pct=0.663265 moved_to_replicas=144548 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[28700, 32771, 98285, 13, 29034, 32773, 144, 40424] avg=32768.0000 max=98285 max_rank=2 min=13 min_rank=3 imbalance_ratio=2.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[28700, 32771, 98285, 13, 29034, 32773, 144, 40424] avg=32768.0000 max=98285 max_rank=2 min=13 min_rank=3 imbalance_ratio=2.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[28700, 32771, 98285, 13, 29034, 32773, 144, 40424] avg=32768.0000 max=98285 max_rank=2 min=13 min_rank=3 imbalance_ratio=2.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[33036, 32088, 33096, 32244, 33096, 32726, 33096, 32762] avg=32768.0000 max=33096 max_rank=2 min=32088 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64467 max_load_reduction_pct=0.660773 moved_to_replicas=193504 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[28700, 32771, 98285, 13, 29034, 32773, 144, 40424] avg=32768.0000 max=98285 max_rank=2 min=13 min_rank=3 imbalance_ratio=2.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[18, 8, 7101, 18891, 33612, 86700, 17685, 98129] avg=32768.0000 max=98129 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.994659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32771, 33096, 32774, 31966, 32773, 32572, 33096] avg=32768.0000 max=33096 max_rank=0 min=31966 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65189 max_load_reduction_pct=0.663265 moved_to_replicas=144548 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[28700, 32771, 98285, 13, 29034, 32773, 144, 40424] avg=32768.0000 max=98285 max_rank=2 min=13 min_rank=3 imbalance_ratio=2.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32771, 33096, 32774, 31966, 32773, 32572, 33096] avg=32768.0000 max=33096 max_rank=0 min=31966 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65189 max_load_reduction_pct=0.663265 moved_to_replicas=144548 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32771, 33096, 32774, 31966, 32773, 32572, 33096] avg=32768.0000 max=33096 max_rank=0 min=31966 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65189 max_load_reduction_pct=0.663265 moved_to_replicas=144548 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[18, 8, 7101, 18891, 33612, 86700, 17685, 98129] avg=32768.0000 max=98129 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.994659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32771, 33096, 32774, 31966, 32773, 32572, 33096] avg=32768.0000 max=33096 max_rank=0 min=31966 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65189 max_load_reduction_pct=0.663265 moved_to_replicas=144548 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[32294, 32765, 33096, 31605, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31605 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65033 max_load_reduction_pct=0.662730 moved_to_replicas=235318 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[28700, 32771, 98285, 13, 29034, 32773, 144, 40424] avg=32768.0000 max=98285 max_rank=2 min=13 min_rank=3 imbalance_ratio=2.999420 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32771, 33096, 32774, 31966, 32773, 32572, 33096] avg=32768.0000 max=33096 max_rank=0 min=31966 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65189 max_load_reduction_pct=0.663265 moved_to_replicas=144548 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[32294, 32765, 33096, 31605, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31605 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65033 max_load_reduction_pct=0.662730 moved_to_replicas=235318 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[18, 8, 7101, 18891, 33612, 86700, 17685, 98129] avg=32768.0000 max=98129 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.994659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[18, 8, 7101, 18891, 33612, 86700, 17685, 98129] avg=32768.0000 max=98129 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.994659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[18, 8, 7101, 18891, 33612, 86700, 17685, 98129] avg=32768.0000 max=98129 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.994659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32771, 33096, 32774, 31966, 32773, 32572, 33096] avg=32768.0000 max=33096 max_rank=0 min=31966 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65189 max_load_reduction_pct=0.663265 moved_to_replicas=144548 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[18, 8, 7101, 18891, 33612, 86700, 17685, 98129] avg=32768.0000 max=98129 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.994659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[65445, 7, 15, 361, 65529, 32500, 65529, 32758] avg=32768.0000 max=65529 max_rank=4 min=7 min_rank=1 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[18, 8, 7101, 18891, 33612, 86700, 17685, 98129] avg=32768.0000 max=98129 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.994659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[32294, 32765, 33096, 31605, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31605 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65033 max_load_reduction_pct=0.662730 moved_to_replicas=235318 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[32294, 32765, 33096, 31605, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31605 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65033 max_load_reduction_pct=0.662730 moved_to_replicas=235318 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[65445, 7, 15, 361, 65529, 32500, 65529, 32758] avg=32768.0000 max=65529 max_rank=4 min=7 min_rank=1 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[32294, 32765, 33096, 31605, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31605 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65033 max_load_reduction_pct=0.662730 moved_to_replicas=235318 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[32294, 32765, 33096, 31605, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31605 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65033 max_load_reduction_pct=0.662730 moved_to_replicas=235318 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[33096, 32440, 32448, 32710, 33096, 32500, 33096, 32758] avg=32768.0000 max=33096 max_rank=0 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=194462 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[32294, 32765, 33096, 31605, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31605 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65033 max_load_reduction_pct=0.662730 moved_to_replicas=235318 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[18, 8, 7101, 18891, 33612, 86700, 17685, 98129] avg=32768.0000 max=98129 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.994659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[33096, 32440, 32448, 32710, 33096, 32500, 33096, 32758] avg=32768.0000 max=33096 max_rank=0 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=194462 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[65445, 7, 15, 361, 65529, 32500, 65529, 32758] avg=32768.0000 max=65529 max_rank=4 min=7 min_rank=1 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[65445, 7, 15, 361, 65529, 32500, 65529, 32758] avg=32768.0000 max=65529 max_rank=4 min=7 min_rank=1 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[65445, 7, 15, 361, 65529, 32500, 65529, 32758] avg=32768.0000 max=65529 max_rank=4 min=7 min_rank=1 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[32294, 32765, 33096, 31605, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=31605 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65033 max_load_reduction_pct=0.662730 moved_to_replicas=235318 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[32745, 65492, 32780, 65559, 32740, 40, 29, 32759] avg=32768.0000 max=65559 max_rank=3 min=29 min_rank=6 imbalance_ratio=2.000702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[65445, 7, 15, 361, 65529, 32500, 65529, 32758] avg=32768.0000 max=65529 max_rank=4 min=7 min_rank=1 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[65445, 7, 15, 361, 65529, 32500, 65529, 32758] avg=32768.0000 max=65529 max_rank=4 min=7 min_rank=1 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[33096, 32440, 32448, 32710, 33096, 32500, 33096, 32758] avg=32768.0000 max=33096 max_rank=0 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=194462 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[32745, 65492, 32780, 65559, 32740, 40, 29, 32759] avg=32768.0000 max=65559 max_rank=3 min=29 min_rank=6 imbalance_ratio=2.000702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[33096, 32440, 32448, 32710, 33096, 32500, 33096, 32758] avg=32768.0000 max=33096 max_rank=0 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=194462 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[33096, 32440, 32448, 32710, 33096, 32500, 33096, 32758] avg=32768.0000 max=33096 max_rank=0 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=194462 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[32745, 33096, 32780, 33096, 32740, 32436, 32492, 32759] avg=32768.0000 max=33096 max_rank=1 min=32436 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32463 max_load_reduction_pct=0.495172 moved_to_replicas=129751 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[33096, 32440, 32448, 32710, 33096, 32500, 33096, 32758] avg=32768.0000 max=33096 max_rank=0 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=194462 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[33096, 32440, 32448, 32710, 33096, 32500, 33096, 32758] avg=32768.0000 max=33096 max_rank=0 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=194462 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[65445, 7, 15, 361, 65529, 32500, 65529, 32758] avg=32768.0000 max=65529 max_rank=4 min=7 min_rank=1 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[32745, 33096, 32780, 33096, 32740, 32436, 32492, 32759] avg=32768.0000 max=33096 max_rank=1 min=32436 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32463 max_load_reduction_pct=0.495172 moved_to_replicas=129751 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[32745, 65492, 32780, 65559, 32740, 40, 29, 32759] avg=32768.0000 max=65559 max_rank=3 min=29 min_rank=6 imbalance_ratio=2.000702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[32745, 65492, 32780, 65559, 32740, 40, 29, 32759] avg=32768.0000 max=65559 max_rank=3 min=29 min_rank=6 imbalance_ratio=2.000702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[32745, 65492, 32780, 65559, 32740, 40, 29, 32759] avg=32768.0000 max=65559 max_rank=3 min=29 min_rank=6 imbalance_ratio=2.000702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[33096, 32440, 32448, 32710, 33096, 32500, 33096, 32758] avg=32768.0000 max=33096 max_rank=0 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=194462 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[58, 32795, 65514, 32742, 32757, 32494, 33026, 32758] avg=32768.0000 max=65514 max_rank=2 min=58 min_rank=0 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[32745, 65492, 32780, 65559, 32740, 40, 29, 32759] avg=32768.0000 max=65559 max_rank=3 min=29 min_rank=6 imbalance_ratio=2.000702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[32745, 65492, 32780, 65559, 32740, 40, 29, 32759] avg=32768.0000 max=65559 max_rank=3 min=29 min_rank=6 imbalance_ratio=2.000702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[32745, 33096, 32780, 33096, 32740, 32436, 32492, 32759] avg=32768.0000 max=33096 max_rank=1 min=32436 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32463 max_load_reduction_pct=0.495172 moved_to_replicas=129751 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[58, 32795, 65514, 32742, 32757, 32494, 33026, 32758] avg=32768.0000 max=65514 max_rank=2 min=58 min_rank=0 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[32745, 33096, 32780, 33096, 32740, 32436, 32492, 32759] avg=32768.0000 max=33096 max_rank=1 min=32436 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32463 max_load_reduction_pct=0.495172 moved_to_replicas=129751 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[32745, 33096, 32780, 33096, 32740, 32436, 32492, 32759] avg=32768.0000 max=33096 max_rank=1 min=32436 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32463 max_load_reduction_pct=0.495172 moved_to_replicas=129751 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[32476, 32795, 33096, 32742, 32757, 32494, 33026, 32758] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=64849 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[32745, 33096, 32780, 33096, 32740, 32436, 32492, 32759] avg=32768.0000 max=33096 max_rank=1 min=32436 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32463 max_load_reduction_pct=0.495172 moved_to_replicas=129751 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[32745, 33096, 32780, 33096, 32740, 32436, 32492, 32759] avg=32768.0000 max=33096 max_rank=1 min=32436 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32463 max_load_reduction_pct=0.495172 moved_to_replicas=129751 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[32476, 32795, 33096, 32742, 32757, 32494, 33026, 32758] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=64849 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[32745, 65492, 32780, 65559, 32740, 40, 29, 32759] avg=32768.0000 max=65559 max_rank=3 min=29 min_rank=6 imbalance_ratio=2.000702 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[58, 32795, 65514, 32742, 32757, 32494, 33026, 32758] avg=32768.0000 max=65514 max_rank=2 min=58 min_rank=0 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[58, 32795, 65514, 32742, 32757, 32494, 33026, 32758] avg=32768.0000 max=65514 max_rank=2 min=58 min_rank=0 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[58, 32795, 65514, 32742, 32757, 32494, 33026, 32758] avg=32768.0000 max=65514 max_rank=2 min=58 min_rank=0 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[37, 152, 68580, 13, 95024, 32782, 32765, 32791] avg=32768.0000 max=95024 max_rank=4 min=13 min_rank=3 imbalance_ratio=2.899902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[32745, 33096, 32780, 33096, 32740, 32436, 32492, 32759] avg=32768.0000 max=33096 max_rank=1 min=32436 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32463 max_load_reduction_pct=0.495172 moved_to_replicas=129751 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[58, 32795, 65514, 32742, 32757, 32494, 33026, 32758] avg=32768.0000 max=65514 max_rank=2 min=58 min_rank=0 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[58, 32795, 65514, 32742, 32757, 32494, 33026, 32758] avg=32768.0000 max=65514 max_rank=2 min=58 min_rank=0 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[37, 152, 68580, 13, 95024, 32782, 32765, 32791] avg=32768.0000 max=95024 max_rank=4 min=13 min_rank=3 imbalance_ratio=2.899902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[32476, 32795, 33096, 32742, 32757, 32494, 33026, 32758] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=64849 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[32476, 32795, 33096, 32742, 32757, 32494, 33026, 32758] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=64849 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[31942, 32908, 33096, 32764, 33096, 32782, 32765, 32791] avg=32768.0000 max=33096 max_rank=2 min=31942 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61928 max_load_reduction_pct=0.651709 moved_to_replicas=194913 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[32476, 32795, 33096, 32742, 32757, 32494, 33026, 32758] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=64849 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[32476, 32795, 33096, 32742, 32757, 32494, 33026, 32758] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=64849 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[32476, 32795, 33096, 32742, 32757, 32494, 33026, 32758] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=64849 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[31942, 32908, 33096, 32764, 33096, 32782, 32765, 32791] avg=32768.0000 max=33096 max_rank=2 min=31942 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61928 max_load_reduction_pct=0.651709 moved_to_replicas=194913 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[58, 32795, 65514, 32742, 32757, 32494, 33026, 32758] avg=32768.0000 max=65514 max_rank=2 min=58 min_rank=0 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[37, 152, 68580, 13, 95024, 32782, 32765, 32791] avg=32768.0000 max=95024 max_rank=4 min=13 min_rank=3 imbalance_ratio=2.899902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[37, 152, 68580, 13, 95024, 32782, 32765, 32791] avg=32768.0000 max=95024 max_rank=4 min=13 min_rank=3 imbalance_ratio=2.899902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[15, 33023, 45, 32771, 66040, 32777, 32566, 64907] avg=32768.0000 max=66040 max_rank=4 min=15 min_rank=0 imbalance_ratio=2.015381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[37, 152, 68580, 13, 95024, 32782, 32765, 32791] avg=32768.0000 max=95024 max_rank=4 min=13 min_rank=3 imbalance_ratio=2.899902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[32476, 32795, 33096, 32742, 32757, 32494, 33026, 32758] avg=32768.0000 max=33096 max_rank=2 min=32476 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=64849 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[37, 152, 68580, 13, 95024, 32782, 32765, 32791] avg=32768.0000 max=95024 max_rank=4 min=13 min_rank=3 imbalance_ratio=2.899902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[37, 152, 68580, 13, 95024, 32782, 32765, 32791] avg=32768.0000 max=95024 max_rank=4 min=13 min_rank=3 imbalance_ratio=2.899902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[15, 33023, 45, 32771, 66040, 32777, 32566, 64907] avg=32768.0000 max=66040 max_rank=4 min=15 min_rank=0 imbalance_ratio=2.015381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[31942, 32908, 33096, 32764, 33096, 32782, 32765, 32791] avg=32768.0000 max=33096 max_rank=2 min=31942 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61928 max_load_reduction_pct=0.651709 moved_to_replicas=194913 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[31942, 32908, 33096, 32764, 33096, 32782, 32765, 32791] avg=32768.0000 max=33096 max_rank=2 min=31942 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61928 max_load_reduction_pct=0.651709 moved_to_replicas=194913 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[32762, 33023, 32053, 32771, 33096, 32777, 32566, 33096] avg=32768.0000 max=33096 max_rank=4 min=32053 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32944 max_load_reduction_pct=0.498849 moved_to_replicas=129472 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[31942, 32908, 33096, 32764, 33096, 32782, 32765, 32791] avg=32768.0000 max=33096 max_rank=2 min=31942 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61928 max_load_reduction_pct=0.651709 moved_to_replicas=194913 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[31942, 32908, 33096, 32764, 33096, 32782, 32765, 32791] avg=32768.0000 max=33096 max_rank=2 min=31942 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61928 max_load_reduction_pct=0.651709 moved_to_replicas=194913 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[31942, 32908, 33096, 32764, 33096, 32782, 32765, 32791] avg=32768.0000 max=33096 max_rank=2 min=31942 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61928 max_load_reduction_pct=0.651709 moved_to_replicas=194913 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[32762, 33023, 32053, 32771, 33096, 32777, 32566, 33096] avg=32768.0000 max=33096 max_rank=4 min=32053 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32944 max_load_reduction_pct=0.498849 moved_to_replicas=129472 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[37, 152, 68580, 13, 95024, 32782, 32765, 32791] avg=32768.0000 max=95024 max_rank=4 min=13 min_rank=3 imbalance_ratio=2.899902 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[15, 33023, 45, 32771, 66040, 32777, 32566, 64907] avg=32768.0000 max=66040 max_rank=4 min=15 min_rank=0 imbalance_ratio=2.015381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[15, 33023, 45, 32771, 66040, 32777, 32566, 64907] avg=32768.0000 max=66040 max_rank=4 min=15 min_rank=0 imbalance_ratio=2.015381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[96329, 32456, 63965, 32766, 19, 3840, 32768, 1] avg=32768.0000 max=96329 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.939728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[15, 33023, 45, 32771, 66040, 32777, 32566, 64907] avg=32768.0000 max=66040 max_rank=4 min=15 min_rank=0 imbalance_ratio=2.015381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[15, 33023, 45, 32771, 66040, 32777, 32566, 64907] avg=32768.0000 max=66040 max_rank=4 min=15 min_rank=0 imbalance_ratio=2.015381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[96329, 32456, 63965, 32766, 19, 3840, 32768, 1] avg=32768.0000 max=96329 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.939728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[15, 33023, 45, 32771, 66040, 32777, 32566, 64907] avg=32768.0000 max=66040 max_rank=4 min=15 min_rank=0 imbalance_ratio=2.015381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[31942, 32908, 33096, 32764, 33096, 32782, 32765, 32791] avg=32768.0000 max=33096 max_rank=2 min=31942 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=61928 max_load_reduction_pct=0.651709 moved_to_replicas=194913 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[32762, 33023, 32053, 32771, 33096, 32777, 32566, 33096] avg=32768.0000 max=33096 max_rank=4 min=32053 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32944 max_load_reduction_pct=0.498849 moved_to_replicas=129472 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[32762, 33023, 32053, 32771, 33096, 32777, 32566, 33096] avg=32768.0000 max=33096 max_rank=4 min=32053 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32944 max_load_reduction_pct=0.498849 moved_to_replicas=129472 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[33096, 32456, 33096, 32766, 32167, 33096, 32768, 32699] avg=32768.0000 max=33096 max_rank=0 min=32167 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63233 max_load_reduction_pct=0.656427 moved_to_replicas=189192 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[32762, 33023, 32053, 32771, 33096, 32777, 32566, 33096] avg=32768.0000 max=33096 max_rank=4 min=32053 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32944 max_load_reduction_pct=0.498849 moved_to_replicas=129472 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[32762, 33023, 32053, 32771, 33096, 32777, 32566, 33096] avg=32768.0000 max=33096 max_rank=4 min=32053 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32944 max_load_reduction_pct=0.498849 moved_to_replicas=129472 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[32762, 33023, 32053, 32771, 33096, 32777, 32566, 33096] avg=32768.0000 max=33096 max_rank=4 min=32053 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32944 max_load_reduction_pct=0.498849 moved_to_replicas=129472 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[33096, 32456, 33096, 32766, 32167, 33096, 32768, 32699] avg=32768.0000 max=33096 max_rank=0 min=32167 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63233 max_load_reduction_pct=0.656427 moved_to_replicas=189192 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[15, 33023, 45, 32771, 66040, 32777, 32566, 64907] avg=32768.0000 max=66040 max_rank=4 min=15 min_rank=0 imbalance_ratio=2.015381 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[96329, 32456, 63965, 32766, 19, 3840, 32768, 1] avg=32768.0000 max=96329 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.939728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[96329, 32456, 63965, 32766, 19, 3840, 32768, 1] avg=32768.0000 max=96329 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.939728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[96329, 32456, 63965, 32766, 19, 3840, 32768, 1] avg=32768.0000 max=96329 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.939728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[96329, 32456, 63965, 32766, 19, 3840, 32768, 1] avg=32768.0000 max=96329 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.939728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[96329, 32456, 63965, 32766, 19, 3840, 32768, 1] avg=32768.0000 max=96329 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.939728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[32762, 33023, 32053, 32771, 33096, 32777, 32566, 33096] avg=32768.0000 max=33096 max_rank=4 min=32053 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32944 max_load_reduction_pct=0.498849 moved_to_replicas=129472 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[33096, 32456, 33096, 32766, 32167, 33096, 32768, 32699] avg=32768.0000 max=33096 max_rank=0 min=32167 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63233 max_load_reduction_pct=0.656427 moved_to_replicas=189192 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[33096, 32456, 33096, 32766, 32167, 33096, 32768, 32699] avg=32768.0000 max=33096 max_rank=0 min=32167 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63233 max_load_reduction_pct=0.656427 moved_to_replicas=189192 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[33096, 32456, 33096, 32766, 32167, 33096, 32768, 32699] avg=32768.0000 max=33096 max_rank=0 min=32167 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63233 max_load_reduction_pct=0.656427 moved_to_replicas=189192 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[33096, 32456, 33096, 32766, 32167, 33096, 32768, 32699] avg=32768.0000 max=33096 max_rank=0 min=32167 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63233 max_load_reduction_pct=0.656427 moved_to_replicas=189192 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[33096, 32456, 33096, 32766, 32167, 33096, 32768, 32699] avg=32768.0000 max=33096 max_rank=0 min=32167 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63233 max_load_reduction_pct=0.656427 moved_to_replicas=189192 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[96329, 32456, 63965, 32766, 19, 3840, 32768, 1] avg=32768.0000 max=96329 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.939728 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[33096, 32456, 33096, 32766, 32167, 33096, 32768, 32699] avg=32768.0000 max=33096 max_rank=0 min=32167 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=63233 max_load_reduction_pct=0.656427 moved_to_replicas=189192 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=32 total_expected_recv=1048789 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56192 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=32 total_expected_recv=1049276 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=55040 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=32 total_expected_recv=1044251 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57984 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=32 total_expected_recv=1052112 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=50176 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=32 total_expected_recv=1048687 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57088 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=32 total_expected_recv=1046850 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57600 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=32 total_expected_recv=1050672 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=54272 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=32 total_expected_recv=1047971 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56192 peak_demand=33097 required_capacity=1.010040 + [2026-08-30 05:11:45] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 555.4 | throughput per GPU (TFLOP/s/GPU): 600.3 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.104177E+00 | loss scale: 1.0 | grad norm: 5.539 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33097, 31649, 33096, 33095, 31919, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=34816 expected_recv=64745 expected_overflow=0 exact_recv_per_rank=[33097, 31649, 33096, 33095, 31919, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=34816 expected_recv=65731 expected_overflow=0 exact_recv_per_rank=[33097, 31649, 33096, 33095, 31919, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=34816 expected_recv=64971 expected_overflow=0 exact_recv_per_rank=[33097, 31649, 33096, 33095, 31919, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=34816 expected_recv=65249 expected_overflow=0 exact_recv_per_rank=[33097, 31649, 33096, 33095, 31919, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 31649, 33096, 33095, 31919, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 31649, 33096, 33095, 31919, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=34816 expected_recv=64745 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=34816 expected_recv=65015 expected_overflow=0 exact_recv_per_rank=[33097, 31649, 33096, 33095, 31919, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=34816 expected_recv=65731 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=34816 expected_recv=64971 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=34816 expected_recv=65249 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=34816 expected_recv=65015 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 32722, 33096, 32277, 32767, 32739, 32352, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 32722, 33096, 32277, 32767, 32739, 32352, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=34816 expected_recv=64512 expected_overflow=0 exact_recv_per_rank=[33095, 32722, 33096, 32277, 32767, 32739, 32352, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=34816 expected_recv=65096 expected_overflow=0 exact_recv_per_rank=[33095, 32722, 33096, 32277, 32767, 32739, 32352, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=34816 expected_recv=65501 expected_overflow=0 exact_recv_per_rank=[33095, 32722, 33096, 32277, 32767, 32739, 32352, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 32722, 33096, 32277, 32767, 32739, 32352, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=34816 expected_recv=64512 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=34816 expected_recv=65096 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=34816 expected_recv=65069 expected_overflow=0 exact_recv_per_rank=[33095, 32722, 33096, 32277, 32767, 32739, 32352, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=34816 expected_recv=65501 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=34816 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[33095, 32722, 33096, 32277, 32767, 32739, 32352, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=34816 expected_recv=65069 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=34816 expected_recv=65535 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=34816 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[32770, 32742, 33097, 32769, 31695, 33096, 33095, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=34816 expected_recv=65558 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32770, 32742, 33097, 32769, 31695, 33096, 33095, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=34816 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[32770, 32742, 33097, 32769, 31695, 33096, 33095, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32770, 32742, 33097, 32769, 31695, 33096, 33095, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=34816 expected_recv=65840 expected_overflow=0 exact_recv_per_rank=[32770, 32742, 33097, 32769, 31695, 33096, 33095, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=34816 expected_recv=65535 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=34816 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[32770, 32742, 33097, 32769, 31695, 33096, 33095, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=34816 expected_recv=62805 expected_overflow=0 exact_recv_per_rank=[32770, 32742, 33097, 32769, 31695, 33096, 33095, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=34816 expected_recv=65976 expected_overflow=0 exact_recv_per_rank=[32770, 32742, 33097, 32769, 31695, 33096, 33095, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=34816 expected_recv=65840 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=34816 expected_recv=66190 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=34816 expected_recv=65241 expected_overflow=0 exact_recv_per_rank=[32756, 32585, 33096, 32745, 33096, 31793, 32977, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=34816 expected_recv=62805 actual_valid_recv=63744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=34816 expected_recv=65976 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=63744 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=34816 expected_recv=65241 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32756, 32585, 33096, 32745, 33096, 31793, 32977, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=34816 expected_recv=65478 expected_overflow=0 exact_recv_per_rank=[32756, 32585, 33096, 32745, 33096, 31793, 32977, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=34816 expected_recv=65478 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=34816 expected_recv=63340 expected_overflow=0 exact_recv_per_rank=[32756, 32585, 33096, 32745, 33096, 31793, 32977, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=34816 expected_recv=65631 expected_overflow=0 exact_recv_per_rank=[32756, 32585, 33096, 32745, 33096, 31793, 32977, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=34816 expected_recv=66022 expected_overflow=0 exact_recv_per_rank=[32756, 32585, 33096, 32745, 33096, 31793, 32977, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32756, 32585, 33096, 32745, 33096, 31793, 32977, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32756, 32585, 33096, 32745, 33096, 31793, 32977, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=34816 expected_recv=63340 actual_valid_recv=63872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=34816 expected_recv=65631 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=34816 expected_recv=66022 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=63872 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=34816 expected_recv=63286 expected_overflow=0 exact_recv_per_rank=[31617, 33097, 33096, 32784, 33094, 32762, 32769, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=34816 expected_recv=63286 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31617, 33097, 33096, 32784, 33094, 32762, 32769, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=34816 expected_recv=65551 expected_overflow=0 exact_recv_per_rank=[31617, 33097, 33096, 32784, 33094, 32762, 32769, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=64384 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=34816 expected_recv=65551 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=34816 expected_recv=65551 expected_overflow=0 exact_recv_per_rank=[31617, 33097, 33096, 32784, 33094, 32762, 32769, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=34816 expected_recv=65784 expected_overflow=0 exact_recv_per_rank=[31617, 33097, 33096, 32784, 33094, 32762, 32769, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31617, 33097, 33096, 32784, 33094, 32762, 32769, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=34816 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[31617, 33097, 33096, 32784, 33094, 32762, 32769, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=34816 expected_recv=65551 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=34816 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[31617, 33097, 33096, 32784, 33094, 32762, 32769, 32925] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=34816 expected_recv=65784 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=34816 expected_recv=65542 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=34816 expected_recv=66190 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31370, 32858, 33094, 32768, 33095, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=34816 expected_recv=62836 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31370, 32858, 33094, 32768, 33095, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=34816 expected_recv=65616 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31370, 32858, 33094, 32768, 33095, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=34816 expected_recv=62836 actual_valid_recv=64000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=34816 expected_recv=65616 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=64000 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=34816 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31370, 32858, 33094, 32768, 33095, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=34816 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31370, 32858, 33094, 32768, 33095, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=34816 expected_recv=65535 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=34816 expected_recv=66188 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31370, 32858, 33094, 32768, 33095, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31370, 32858, 33094, 32768, 33095, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31370, 32858, 33094, 32768, 33095, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=34816 expected_recv=65537 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=34816 expected_recv=66188 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=34816 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[32768, 32757, 32489, 33097, 33096, 32766, 32407, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=34816 expected_recv=64961 expected_overflow=0 exact_recv_per_rank=[32768, 32757, 32489, 33097, 33096, 32766, 32407, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32768, 32757, 32489, 33097, 33096, 32766, 32407, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=34816 expected_recv=65543 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=34816 expected_recv=64961 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=34816 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[32768, 32757, 32489, 33097, 33096, 32766, 32407, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32768, 32757, 32489, 33097, 33096, 32766, 32407, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=34816 expected_recv=65514 expected_overflow=0 exact_recv_per_rank=[32768, 32757, 32489, 33097, 33096, 32766, 32407, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=34816 expected_recv=65537 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=34816 expected_recv=65513 expected_overflow=0 exact_recv_per_rank=[32768, 32757, 32489, 33097, 33096, 32766, 32407, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=34816 expected_recv=65514 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=34816 expected_recv=64836 expected_overflow=0 exact_recv_per_rank=[32768, 32757, 32489, 33097, 33096, 32766, 32407, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32437, 33096, 32464, 33096, 32758, 32753, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=34816 expected_recv=65513 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32437, 33096, 32464, 33096, 32758, 32753, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=34816 expected_recv=64836 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=34816 expected_recv=64956 expected_overflow=0 exact_recv_per_rank=[33096, 32437, 33096, 32464, 33096, 32758, 32753, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=34816 expected_recv=64956 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=34816 expected_recv=65446 expected_overflow=0 exact_recv_per_rank=[33096, 32437, 33096, 32464, 33096, 32758, 32753, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32437, 33096, 32464, 33096, 32758, 32753, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=34816 expected_recv=65446 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=34816 expected_recv=64895 expected_overflow=0 exact_recv_per_rank=[33096, 32437, 33096, 32464, 33096, 32758, 32753, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=34816 expected_recv=64895 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=34816 expected_recv=64912 expected_overflow=0 exact_recv_per_rank=[33096, 32437, 33096, 32464, 33096, 32758, 32753, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=34816 expected_recv=65503 expected_overflow=0 exact_recv_per_rank=[33096, 32437, 33096, 32464, 33096, 32758, 32753, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 31127, 33096, 33098, 33096, 32769, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 31127, 33096, 33098, 33096, 32769, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=34816 expected_recv=64912 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=34816 expected_recv=65503 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=34816 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33095, 31127, 33096, 33098, 33096, 32769, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=34816 expected_recv=66194 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=34816 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[33095, 31127, 33096, 33098, 33096, 32769, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 31127, 33096, 33098, 33096, 32769, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=34816 expected_recv=65531 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 31127, 33096, 33098, 33096, 32769, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=34816 expected_recv=65555 expected_overflow=0 exact_recv_per_rank=[33095, 31127, 33096, 33098, 33096, 32769, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=34816 expected_recv=62242 expected_overflow=0 exact_recv_per_rank=[33095, 31127, 33096, 33098, 33096, 32769, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=34816 expected_recv=64669 expected_overflow=0 exact_recv_per_rank=[32417, 32768, 32505, 32349, 33096, 32817, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=34816 expected_recv=64192 expected_overflow=0 exact_recv_per_rank=[32417, 32768, 32505, 32349, 33096, 32817, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=34816 expected_recv=65399 expected_overflow=0 exact_recv_per_rank=[32417, 32768, 32505, 32349, 33096, 32817, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=34816 expected_recv=65555 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=34816 expected_recv=62242 actual_valid_recv=62976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=34816 expected_recv=64669 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=34816 expected_recv=64192 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=34816 expected_recv=65399 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=34816 expected_recv=65913 expected_overflow=0 exact_recv_per_rank=[32417, 32768, 32505, 32349, 33096, 32817, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=62976 slack_slots=6656 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=64896 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32417, 32768, 32505, 32349, 33096, 32817, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=34816 expected_recv=65913 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32417, 32768, 32505, 32349, 33096, 32817, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32417, 32768, 32505, 32349, 33096, 32817, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 32281, 33096, 33096, 32276, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=34816 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[32417, 32768, 32505, 32349, 33096, 32817, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=34816 expected_recv=64425 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 32281, 33096, 33096, 32276, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 32281, 33096, 33096, 32276, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=34816 expected_recv=65539 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=34816 expected_recv=64425 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=34816 expected_recv=64687 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 32281, 33096, 33096, 32276, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 32281, 33096, 33096, 32276, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=34816 expected_recv=64687 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=34816 expected_recv=65518 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 32281, 33096, 33096, 32276, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=34816 expected_recv=65518 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=34816 expected_recv=65500 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 32768, 33096, 32758, 32768, 32367, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 32281, 33096, 33096, 32276, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=34816 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 32768, 33096, 32758, 32768, 32367, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=34816 expected_recv=64890 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 32281, 33096, 33096, 32276, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 32768, 33096, 32758, 32768, 32367, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=34816 expected_recv=65500 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=34816 expected_recv=65536 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=34816 expected_recv=64890 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=34816 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 32768, 33096, 32758, 32768, 32367, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=34816 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 32768, 33096, 32758, 32768, 32367, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=34816 expected_recv=65238 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 32768, 33096, 32758, 32768, 32367, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=34816 expected_recv=65539 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=34816 expected_recv=65526 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=34816 expected_recv=65238 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=34816 expected_recv=65567 expected_overflow=0 exact_recv_per_rank=[32772, 32742, 33096, 32771, 32767, 32268, 32969, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=34816 expected_recv=65567 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=34816 expected_recv=64565 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 32768, 33096, 32758, 32768, 32367, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32772, 32742, 33096, 32771, 32767, 32268, 32969, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=34816 expected_recv=65548 expected_overflow=0 exact_recv_per_rank=[32772, 32742, 33096, 32771, 32767, 32268, 32969, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32759, 33096, 32768, 33096, 32758, 32768, 32367, 32532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=34816 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[32772, 32742, 33096, 32771, 32767, 32268, 32969, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=34816 expected_recv=64387 expected_overflow=0 exact_recv_per_rank=[32772, 32742, 33096, 32771, 32767, 32268, 32969, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=34816 expected_recv=64565 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=34816 expected_recv=65500 expected_overflow=0 exact_recv_per_rank=[32772, 32742, 33096, 32771, 32767, 32268, 32969, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=34816 expected_recv=65526 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=34816 expected_recv=65548 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=34816 expected_recv=64387 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=34816 expected_recv=65500 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=64896 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=34816 expected_recv=63582 expected_overflow=0 exact_recv_per_rank=[31789, 33094, 33097, 32435, 33096, 32769, 32767, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=34816 expected_recv=63582 actual_valid_recv=64128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=64128 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31789, 33094, 33097, 32435, 33096, 32769, 32767, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=34816 expected_recv=66065 expected_overflow=0 exact_recv_per_rank=[32772, 32742, 33096, 32771, 32767, 32268, 32969, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=34816 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[31789, 33094, 33097, 32435, 33096, 32769, 32767, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=34816 expected_recv=66065 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=34816 expected_recv=64859 expected_overflow=0 exact_recv_per_rank=[31789, 33094, 33097, 32435, 33096, 32769, 32767, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=34816 expected_recv=65503 expected_overflow=0 exact_recv_per_rank=[32772, 32742, 33096, 32771, 32767, 32268, 32969, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=34816 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[31789, 33094, 33097, 32435, 33096, 32769, 32767, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=34816 expected_recv=66194 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31789, 33094, 33097, 32435, 33096, 32769, 32767, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=34816 expected_recv=64859 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=34816 expected_recv=65503 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=34816 expected_recv=65545 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=34816 expected_recv=65386 expected_overflow=0 exact_recv_per_rank=[32623, 33096, 32371, 32766, 33096, 32769, 32327, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=34816 expected_recv=65386 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32623, 33096, 32371, 32766, 33096, 32769, 32327, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=34816 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[31789, 33094, 33097, 32435, 33096, 32769, 32767, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=34816 expected_recv=64475 expected_overflow=0 exact_recv_per_rank=[32623, 33096, 32371, 32766, 33096, 32769, 32327, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=34816 expected_recv=65534 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=34816 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[32623, 33096, 32371, 32766, 33096, 32769, 32327, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=34816 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[32623, 33096, 32371, 32766, 33096, 32769, 32327, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=34816 expected_recv=66189 expected_overflow=0 exact_recv_per_rank=[31789, 33094, 33097, 32435, 33096, 32769, 32767, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=34816 expected_recv=64475 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32623, 33096, 32371, 32766, 33096, 32769, 32327, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=34816 expected_recv=65536 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=34816 expected_recv=65542 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32903, 32767, 32757, 32730, 32768, 32361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=34816 expected_recv=66189 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=67200 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=34816 expected_recv=65121 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32903, 32767, 32757, 32730, 32768, 32361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=34816 expected_recv=64773 expected_overflow=0 exact_recv_per_rank=[32623, 33096, 32371, 32766, 33096, 32769, 32327, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=34816 expected_recv=65121 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=34816 expected_recv=65999 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32903, 32767, 32757, 32730, 32768, 32361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=34816 expected_recv=64773 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=34816 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32903, 32767, 32757, 32730, 32768, 32361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=34816 expected_recv=65280 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32903, 32767, 32757, 32730, 32768, 32361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32623, 33096, 32371, 32766, 33096, 32769, 32327, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=34816 expected_recv=65999 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=34816 expected_recv=65100 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32903, 32767, 32757, 32730, 32768, 32361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65024 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[37504, 30969, 24083, 26182, 5525, 26998, 42602, 68281] avg=32768.0000 max=68281 max_rank=7 min=5525 min_rank=4 imbalance_ratio=2.083771 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=34816 expected_recv=65533 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=34816 expected_recv=65280 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=34816 expected_recv=65100 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[33097, 31649, 33096, 33095, 31919, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31649 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=93983 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[37504, 30969, 24083, 26182, 5525, 26998, 42602, 68281] avg=32768.0000 max=68281 max_rank=7 min=5525 min_rank=4 imbalance_ratio=2.083771 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=34816 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32903, 32767, 32757, 32730, 32768, 32361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=34816 expected_recv=65536 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[37504, 30969, 24083, 26182, 5525, 26998, 42602, 68281] avg=32768.0000 max=68281 max_rank=7 min=5525 min_rank=4 imbalance_ratio=2.083771 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=34816 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32903, 32767, 32757, 32730, 32768, 32361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[37504, 30969, 24083, 26182, 5525, 26998, 42602, 68281] avg=32768.0000 max=68281 max_rank=7 min=5525 min_rank=4 imbalance_ratio=2.083771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[37504, 30969, 24083, 26182, 5525, 26998, 42602, 68281] avg=32768.0000 max=68281 max_rank=7 min=5525 min_rank=4 imbalance_ratio=2.083771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[37504, 30969, 24083, 26182, 5525, 26998, 42602, 68281] avg=32768.0000 max=68281 max_rank=7 min=5525 min_rank=4 imbalance_ratio=2.083771 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=34816 expected_recv=65527 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[33097, 31649, 33096, 33095, 31919, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31649 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=93983 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[4464, 32722, 99988, 28024, 32767, 1, 30915, 33263] avg=32768.0000 max=99988 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.051392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[33097, 31649, 33096, 33095, 31919, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31649 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=93983 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[33097, 31649, 33096, 33095, 31919, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31649 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=93983 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[33097, 31649, 33096, 33095, 31919, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31649 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=93983 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[33097, 31649, 33096, 33095, 31919, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31649 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=93983 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[33095, 32722, 33096, 32277, 32767, 32739, 32352, 33096] avg=32768.0000 max=33096 max_rank=2 min=32277 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=66892 max_load_reduction_pct=0.669000 moved_to_replicas=134044 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[37504, 30969, 24083, 26182, 5525, 26998, 42602, 68281] avg=32768.0000 max=68281 max_rank=7 min=5525 min_rank=4 imbalance_ratio=2.083771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[4464, 32722, 99988, 28024, 32767, 1, 30915, 33263] avg=32768.0000 max=99988 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.051392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[37504, 30969, 24083, 26182, 5525, 26998, 42602, 68281] avg=32768.0000 max=68281 max_rank=7 min=5525 min_rank=4 imbalance_ratio=2.083771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[4464, 32722, 99988, 28024, 32767, 1, 30915, 33263] avg=32768.0000 max=99988 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.051392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[4464, 32722, 99988, 28024, 32767, 1, 30915, 33263] avg=32768.0000 max=99988 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.051392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[33095, 32722, 33096, 32277, 32767, 32739, 32352, 33096] avg=32768.0000 max=33096 max_rank=2 min=32277 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=66892 max_load_reduction_pct=0.669000 moved_to_replicas=134044 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[4464, 32722, 99988, 28024, 32767, 1, 30915, 33263] avg=32768.0000 max=99988 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.051392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[4464, 32722, 99988, 28024, 32767, 1, 30915, 33263] avg=32768.0000 max=99988 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.051392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[33097, 31649, 33096, 33095, 31919, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31649 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=93983 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[32770, 28676, 55866, 32769, 3, 64788, 14392, 32880] avg=32768.0000 max=64788 max_rank=5 min=3 min_rank=4 imbalance_ratio=1.977173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[33097, 31649, 33096, 33095, 31919, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31649 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35184 max_load_reduction_pct=0.515282 moved_to_replicas=93983 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[33095, 32722, 33096, 32277, 32767, 32739, 32352, 33096] avg=32768.0000 max=33096 max_rank=2 min=32277 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=66892 max_load_reduction_pct=0.669000 moved_to_replicas=134044 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[33095, 32722, 33096, 32277, 32767, 32739, 32352, 33096] avg=32768.0000 max=33096 max_rank=2 min=32277 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=66892 max_load_reduction_pct=0.669000 moved_to_replicas=134044 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[33095, 32722, 33096, 32277, 32767, 32739, 32352, 33096] avg=32768.0000 max=33096 max_rank=2 min=32277 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=66892 max_load_reduction_pct=0.669000 moved_to_replicas=134044 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[33095, 32722, 33096, 32277, 32767, 32739, 32352, 33096] avg=32768.0000 max=33096 max_rank=2 min=32277 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=66892 max_load_reduction_pct=0.669000 moved_to_replicas=134044 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[32770, 28676, 55866, 32769, 3, 64788, 14392, 32880] avg=32768.0000 max=64788 max_rank=5 min=3 min_rank=4 imbalance_ratio=1.977173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[32770, 32742, 33097, 32769, 31695, 33096, 33095, 32880] avg=32768.0000 max=33097 max_rank=2 min=31695 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31691 max_load_reduction_pct=0.489149 moved_to_replicas=106099 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[4464, 32722, 99988, 28024, 32767, 1, 30915, 33263] avg=32768.0000 max=99988 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.051392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[4464, 32722, 99988, 28024, 32767, 1, 30915, 33263] avg=32768.0000 max=99988 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.051392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[32770, 28676, 55866, 32769, 3, 64788, 14392, 32880] avg=32768.0000 max=64788 max_rank=5 min=3 min_rank=4 imbalance_ratio=1.977173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[32770, 32742, 33097, 32769, 31695, 33096, 33095, 32880] avg=32768.0000 max=33097 max_rank=2 min=31695 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31691 max_load_reduction_pct=0.489149 moved_to_replicas=106099 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[32770, 28676, 55866, 32769, 3, 64788, 14392, 32880] avg=32768.0000 max=64788 max_rank=5 min=3 min_rank=4 imbalance_ratio=1.977173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[32770, 28676, 55866, 32769, 3, 64788, 14392, 32880] avg=32768.0000 max=64788 max_rank=5 min=3 min_rank=4 imbalance_ratio=1.977173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[32770, 28676, 55866, 32769, 3, 64788, 14392, 32880] avg=32768.0000 max=64788 max_rank=5 min=3 min_rank=4 imbalance_ratio=1.977173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[2, 32585, 33657, 0, 67431, 31232, 32977, 64260] avg=32768.0000 max=67431 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.057831 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[33095, 32722, 33096, 32277, 32767, 32739, 32352, 33096] avg=32768.0000 max=33096 max_rank=2 min=32277 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=66892 max_load_reduction_pct=0.669000 moved_to_replicas=134044 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[32770, 32742, 33097, 32769, 31695, 33096, 33095, 32880] avg=32768.0000 max=33097 max_rank=2 min=31695 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31691 max_load_reduction_pct=0.489149 moved_to_replicas=106099 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[33095, 32722, 33096, 32277, 32767, 32739, 32352, 33096] avg=32768.0000 max=33096 max_rank=2 min=32277 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=66892 max_load_reduction_pct=0.669000 moved_to_replicas=134044 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[32770, 32742, 33097, 32769, 31695, 33096, 33095, 32880] avg=32768.0000 max=33097 max_rank=2 min=31695 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31691 max_load_reduction_pct=0.489149 moved_to_replicas=106099 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[32770, 32742, 33097, 32769, 31695, 33096, 33095, 32880] avg=32768.0000 max=33097 max_rank=2 min=31695 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31691 max_load_reduction_pct=0.489149 moved_to_replicas=106099 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[32770, 32742, 33097, 32769, 31695, 33096, 33095, 32880] avg=32768.0000 max=33097 max_rank=2 min=31695 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31691 max_load_reduction_pct=0.489149 moved_to_replicas=106099 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[2, 32585, 33657, 0, 67431, 31232, 32977, 64260] avg=32768.0000 max=67431 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.057831 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[32756, 32585, 33096, 32745, 33096, 31793, 32977, 33096] avg=32768.0000 max=33096 max_rank=2 min=31793 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34335 max_load_reduction_pct=0.509187 moved_to_replicas=131626 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[32770, 28676, 55866, 32769, 3, 64788, 14392, 32880] avg=32768.0000 max=64788 max_rank=5 min=3 min_rank=4 imbalance_ratio=1.977173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[2, 32585, 33657, 0, 67431, 31232, 32977, 64260] avg=32768.0000 max=67431 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.057831 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[32756, 32585, 33096, 32745, 33096, 31793, 32977, 33096] avg=32768.0000 max=33096 max_rank=2 min=31793 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34335 max_load_reduction_pct=0.509187 moved_to_replicas=131626 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[32770, 28676, 55866, 32769, 3, 64788, 14392, 32880] avg=32768.0000 max=64788 max_rank=5 min=3 min_rank=4 imbalance_ratio=1.977173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[2, 32585, 33657, 0, 67431, 31232, 32977, 64260] avg=32768.0000 max=67431 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.057831 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[2, 32585, 33657, 0, 67431, 31232, 32977, 64260] avg=32768.0000 max=67431 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.057831 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[2, 32585, 33657, 0, 67431, 31232, 32977, 64260] avg=32768.0000 max=67431 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.057831 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[26536, 70985, 65363, 32784, 786, 32762, 32769, 159] avg=32768.0000 max=70985 max_rank=1 min=159 min_rank=7 imbalance_ratio=2.166290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[32770, 32742, 33097, 32769, 31695, 33096, 33095, 32880] avg=32768.0000 max=33097 max_rank=2 min=31695 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31691 max_load_reduction_pct=0.489149 moved_to_replicas=106099 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[32756, 32585, 33096, 32745, 33096, 31793, 32977, 33096] avg=32768.0000 max=33096 max_rank=2 min=31793 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34335 max_load_reduction_pct=0.509187 moved_to_replicas=131626 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[32770, 32742, 33097, 32769, 31695, 33096, 33095, 32880] avg=32768.0000 max=33097 max_rank=2 min=31695 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31691 max_load_reduction_pct=0.489149 moved_to_replicas=106099 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[32756, 32585, 33096, 32745, 33096, 31793, 32977, 33096] avg=32768.0000 max=33096 max_rank=2 min=31793 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34335 max_load_reduction_pct=0.509187 moved_to_replicas=131626 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[32756, 32585, 33096, 32745, 33096, 31793, 32977, 33096] avg=32768.0000 max=33096 max_rank=2 min=31793 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34335 max_load_reduction_pct=0.509187 moved_to_replicas=131626 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[32756, 32585, 33096, 32745, 33096, 31793, 32977, 33096] avg=32768.0000 max=33096 max_rank=2 min=31793 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34335 max_load_reduction_pct=0.509187 moved_to_replicas=131626 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[26536, 70985, 65363, 32784, 786, 32762, 32769, 159] avg=32768.0000 max=70985 max_rank=1 min=159 min_rank=7 imbalance_ratio=2.166290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[31617, 33097, 33096, 32784, 33094, 32762, 32769, 32925] avg=32768.0000 max=33097 max_rank=1 min=31617 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37888 max_load_reduction_pct=0.533747 moved_to_replicas=143044 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[2, 32585, 33657, 0, 67431, 31232, 32977, 64260] avg=32768.0000 max=67431 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.057831 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[26536, 70985, 65363, 32784, 786, 32762, 32769, 159] avg=32768.0000 max=70985 max_rank=1 min=159 min_rank=7 imbalance_ratio=2.166290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[31617, 33097, 33096, 32784, 33094, 32762, 32769, 32925] avg=32768.0000 max=33097 max_rank=1 min=31617 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37888 max_load_reduction_pct=0.533747 moved_to_replicas=143044 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[26536, 70985, 65363, 32784, 786, 32762, 32769, 159] avg=32768.0000 max=70985 max_rank=1 min=159 min_rank=7 imbalance_ratio=2.166290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[2, 32585, 33657, 0, 67431, 31232, 32977, 64260] avg=32768.0000 max=67431 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.057831 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[26536, 70985, 65363, 32784, 786, 32762, 32769, 159] avg=32768.0000 max=70985 max_rank=1 min=159 min_rank=7 imbalance_ratio=2.166290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[26536, 70985, 65363, 32784, 786, 32762, 32769, 159] avg=32768.0000 max=70985 max_rank=1 min=159 min_rank=7 imbalance_ratio=2.166290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[65365, 83353, 3606, 32858, 5109, 32768, 39084, 1] avg=32768.0000 max=83353 max_rank=1 min=1 min_rank=7 imbalance_ratio=2.543732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[32756, 32585, 33096, 32745, 33096, 31793, 32977, 33096] avg=32768.0000 max=33096 max_rank=2 min=31793 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34335 max_load_reduction_pct=0.509187 moved_to_replicas=131626 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[31617, 33097, 33096, 32784, 33094, 32762, 32769, 32925] avg=32768.0000 max=33097 max_rank=1 min=31617 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37888 max_load_reduction_pct=0.533747 moved_to_replicas=143044 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[31617, 33097, 33096, 32784, 33094, 32762, 32769, 32925] avg=32768.0000 max=33097 max_rank=1 min=31617 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37888 max_load_reduction_pct=0.533747 moved_to_replicas=143044 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[31617, 33097, 33096, 32784, 33094, 32762, 32769, 32925] avg=32768.0000 max=33097 max_rank=1 min=31617 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37888 max_load_reduction_pct=0.533747 moved_to_replicas=143044 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[32756, 32585, 33096, 32745, 33096, 31793, 32977, 33096] avg=32768.0000 max=33096 max_rank=2 min=31793 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34335 max_load_reduction_pct=0.509187 moved_to_replicas=131626 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[65365, 83353, 3606, 32858, 5109, 32768, 39084, 1] avg=32768.0000 max=83353 max_rank=1 min=1 min_rank=7 imbalance_ratio=2.543732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[31617, 33097, 33096, 32784, 33094, 32762, 32769, 32925] avg=32768.0000 max=33097 max_rank=1 min=31617 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37888 max_load_reduction_pct=0.533747 moved_to_replicas=143044 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 31370, 32858, 33094, 32768, 33095, 32767] avg=32768.0000 max=33096 max_rank=0 min=31370 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50257 max_load_reduction_pct=0.602942 moved_to_replicas=176609 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 31370, 32858, 33094, 32768, 33095, 32767] avg=32768.0000 max=33096 max_rank=0 min=31370 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50257 max_load_reduction_pct=0.602942 moved_to_replicas=176609 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[26536, 70985, 65363, 32784, 786, 32762, 32769, 159] avg=32768.0000 max=70985 max_rank=1 min=159 min_rank=7 imbalance_ratio=2.166290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[65365, 83353, 3606, 32858, 5109, 32768, 39084, 1] avg=32768.0000 max=83353 max_rank=1 min=1 min_rank=7 imbalance_ratio=2.543732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[65365, 83353, 3606, 32858, 5109, 32768, 39084, 1] avg=32768.0000 max=83353 max_rank=1 min=1 min_rank=7 imbalance_ratio=2.543732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[65365, 83353, 3606, 32858, 5109, 32768, 39084, 1] avg=32768.0000 max=83353 max_rank=1 min=1 min_rank=7 imbalance_ratio=2.543732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[26536, 70985, 65363, 32784, 786, 32762, 32769, 159] avg=32768.0000 max=70985 max_rank=1 min=159 min_rank=7 imbalance_ratio=2.166290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[65365, 83353, 3606, 32858, 5109, 32768, 39084, 1] avg=32768.0000 max=83353 max_rank=1 min=1 min_rank=7 imbalance_ratio=2.543732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32757, 15463, 50123, 65501, 32766, 2, 32764] avg=32768.0000 max=65501 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[31617, 33097, 33096, 32784, 33094, 32762, 32769, 32925] avg=32768.0000 max=33097 max_rank=1 min=31617 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37888 max_load_reduction_pct=0.533747 moved_to_replicas=143044 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 31370, 32858, 33094, 32768, 33095, 32767] avg=32768.0000 max=33096 max_rank=0 min=31370 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50257 max_load_reduction_pct=0.602942 moved_to_replicas=176609 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 31370, 32858, 33094, 32768, 33095, 32767] avg=32768.0000 max=33096 max_rank=0 min=31370 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50257 max_load_reduction_pct=0.602942 moved_to_replicas=176609 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 31370, 32858, 33094, 32768, 33095, 32767] avg=32768.0000 max=33096 max_rank=0 min=31370 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50257 max_load_reduction_pct=0.602942 moved_to_replicas=176609 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32757, 15463, 50123, 65501, 32766, 2, 32764] avg=32768.0000 max=65501 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[31617, 33097, 33096, 32784, 33094, 32762, 32769, 32925] avg=32768.0000 max=33097 max_rank=1 min=31617 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=37888 max_load_reduction_pct=0.533747 moved_to_replicas=143044 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 31370, 32858, 33094, 32768, 33095, 32767] avg=32768.0000 max=33096 max_rank=0 min=31370 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50257 max_load_reduction_pct=0.602942 moved_to_replicas=176609 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32757, 32489, 33097, 33096, 32766, 32407, 32764] avg=32768.0000 max=33097 max_rank=3 min=32407 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32404 max_load_reduction_pct=0.494710 moved_to_replicas=101895 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32757, 32489, 33097, 33096, 32766, 32407, 32764] avg=32768.0000 max=33097 max_rank=3 min=32407 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32404 max_load_reduction_pct=0.494710 moved_to_replicas=101895 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[65365, 83353, 3606, 32858, 5109, 32768, 39084, 1] avg=32768.0000 max=83353 max_rank=1 min=1 min_rank=7 imbalance_ratio=2.543732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32757, 15463, 50123, 65501, 32766, 2, 32764] avg=32768.0000 max=65501 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32757, 15463, 50123, 65501, 32766, 2, 32764] avg=32768.0000 max=65501 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32757, 15463, 50123, 65501, 32766, 2, 32764] avg=32768.0000 max=65501 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[65365, 83353, 3606, 32858, 5109, 32768, 39084, 1] avg=32768.0000 max=83353 max_rank=1 min=1 min_rank=7 imbalance_ratio=2.543732 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32757, 15463, 50123, 65501, 32766, 2, 32764] avg=32768.0000 max=65501 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[65495, 38, 65526, 34, 65535, 32758, 32753, 5] avg=32768.0000 max=65535 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 31370, 32858, 33094, 32768, 33095, 32767] avg=32768.0000 max=33096 max_rank=0 min=31370 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50257 max_load_reduction_pct=0.602942 moved_to_replicas=176609 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32757, 32489, 33097, 33096, 32766, 32407, 32764] avg=32768.0000 max=33097 max_rank=3 min=32407 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32404 max_load_reduction_pct=0.494710 moved_to_replicas=101895 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32757, 32489, 33097, 33096, 32766, 32407, 32764] avg=32768.0000 max=33097 max_rank=3 min=32407 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32404 max_load_reduction_pct=0.494710 moved_to_replicas=101895 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32757, 32489, 33097, 33096, 32766, 32407, 32764] avg=32768.0000 max=33097 max_rank=3 min=32407 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32404 max_load_reduction_pct=0.494710 moved_to_replicas=101895 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[65495, 38, 65526, 34, 65535, 32758, 32753, 5] avg=32768.0000 max=65535 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33096, 31370, 32858, 33094, 32768, 33095, 32767] avg=32768.0000 max=33096 max_rank=0 min=31370 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=50257 max_load_reduction_pct=0.602942 moved_to_replicas=176609 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32757, 32489, 33097, 33096, 32766, 32407, 32764] avg=32768.0000 max=33097 max_rank=3 min=32407 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32404 max_load_reduction_pct=0.494710 moved_to_replicas=101895 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32437, 33096, 32464, 33096, 32758, 32753, 32444] avg=32768.0000 max=33096 max_rank=0 min=32437 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=194508 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32437, 33096, 32464, 33096, 32758, 32753, 32444] avg=32768.0000 max=33096 max_rank=0 min=32437 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=194508 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32757, 15463, 50123, 65501, 32766, 2, 32764] avg=32768.0000 max=65501 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[65495, 38, 65526, 34, 65535, 32758, 32753, 5] avg=32768.0000 max=65535 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[65495, 38, 65526, 34, 65535, 32758, 32753, 5] avg=32768.0000 max=65535 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[65495, 38, 65526, 34, 65535, 32758, 32753, 5] avg=32768.0000 max=65535 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32757, 15463, 50123, 65501, 32766, 2, 32764] avg=32768.0000 max=65501 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[65495, 38, 65526, 34, 65535, 32758, 32753, 5] avg=32768.0000 max=65535 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[11716, 29680, 98301, 24629, 4636, 32769, 2, 60411] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32757, 32489, 33097, 33096, 32766, 32407, 32764] avg=32768.0000 max=33097 max_rank=3 min=32407 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32404 max_load_reduction_pct=0.494710 moved_to_replicas=101895 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32437, 33096, 32464, 33096, 32758, 32753, 32444] avg=32768.0000 max=33096 max_rank=0 min=32437 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=194508 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32437, 33096, 32464, 33096, 32758, 32753, 32444] avg=32768.0000 max=33096 max_rank=0 min=32437 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=194508 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[11716, 29680, 98301, 24629, 4636, 32769, 2, 60411] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32757, 32489, 33097, 33096, 32766, 32407, 32764] avg=32768.0000 max=33097 max_rank=3 min=32407 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32404 max_load_reduction_pct=0.494710 moved_to_replicas=101895 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32437, 33096, 32464, 33096, 32758, 32753, 32444] avg=32768.0000 max=33096 max_rank=0 min=32437 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=194508 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[33095, 31127, 33096, 33098, 33096, 32769, 32768, 33095] avg=32768.0000 max=33098 max_rank=3 min=31127 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=65203 max_load_reduction_pct=0.663299 moved_to_replicas=182169 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32437, 33096, 32464, 33096, 32758, 32753, 32444] avg=32768.0000 max=33096 max_rank=0 min=32437 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=194508 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[33095, 31127, 33096, 33098, 33096, 32769, 32768, 33095] avg=32768.0000 max=33098 max_rank=3 min=31127 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=65203 max_load_reduction_pct=0.663299 moved_to_replicas=182169 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[65495, 38, 65526, 34, 65535, 32758, 32753, 5] avg=32768.0000 max=65535 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[11716, 29680, 98301, 24629, 4636, 32769, 2, 60411] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[11716, 29680, 98301, 24629, 4636, 32769, 2, 60411] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[65495, 38, 65526, 34, 65535, 32758, 32753, 5] avg=32768.0000 max=65535 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[11716, 29680, 98301, 24629, 4636, 32769, 2, 60411] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[11, 2, 32505, 29061, 67818, 32817, 1662, 98268] avg=32768.0000 max=98268 max_rank=7 min=2 min_rank=1 imbalance_ratio=2.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32437, 33096, 32464, 33096, 32758, 32753, 32444] avg=32768.0000 max=33096 max_rank=0 min=32437 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=194508 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[33095, 31127, 33096, 33098, 33096, 32769, 32768, 33095] avg=32768.0000 max=33098 max_rank=3 min=31127 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=65203 max_load_reduction_pct=0.663299 moved_to_replicas=182169 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[11716, 29680, 98301, 24629, 4636, 32769, 2, 60411] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[33095, 31127, 33096, 33098, 33096, 32769, 32768, 33095] avg=32768.0000 max=33098 max_rank=3 min=31127 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=65203 max_load_reduction_pct=0.663299 moved_to_replicas=182169 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[11, 2, 32505, 29061, 67818, 32817, 1662, 98268] avg=32768.0000 max=98268 max_rank=7 min=2 min_rank=1 imbalance_ratio=2.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32437, 33096, 32464, 33096, 32758, 32753, 32444] avg=32768.0000 max=33096 max_rank=0 min=32437 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=194508 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[33095, 31127, 33096, 33098, 33096, 32769, 32768, 33095] avg=32768.0000 max=33098 max_rank=3 min=31127 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=65203 max_load_reduction_pct=0.663299 moved_to_replicas=182169 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[32417, 32768, 32505, 32349, 33096, 32817, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32349 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=200892 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[33095, 31127, 33096, 33098, 33096, 32769, 32768, 33095] avg=32768.0000 max=33098 max_rank=3 min=31127 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=65203 max_load_reduction_pct=0.663299 moved_to_replicas=182169 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[32417, 32768, 32505, 32349, 33096, 32817, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32349 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=200892 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[11, 2, 32505, 29061, 67818, 32817, 1662, 98268] avg=32768.0000 max=98268 max_rank=7 min=2 min_rank=1 imbalance_ratio=2.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[11716, 29680, 98301, 24629, 4636, 32769, 2, 60411] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[11, 2, 32505, 29061, 67818, 32817, 1662, 98268] avg=32768.0000 max=98268 max_rank=7 min=2 min_rank=1 imbalance_ratio=2.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[11716, 29680, 98301, 24629, 4636, 32769, 2, 60411] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[11, 2, 32505, 29061, 67818, 32817, 1662, 98268] avg=32768.0000 max=98268 max_rank=7 min=2 min_rank=1 imbalance_ratio=2.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[64592, 2, 2, 33726, 65536, 150, 65375, 32761] avg=32768.0000 max=65536 max_rank=4 min=2 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[32417, 32768, 32505, 32349, 33096, 32817, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32349 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=200892 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[33095, 31127, 33096, 33098, 33096, 32769, 32768, 33095] avg=32768.0000 max=33098 max_rank=3 min=31127 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=65203 max_load_reduction_pct=0.663299 moved_to_replicas=182169 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[11, 2, 32505, 29061, 67818, 32817, 1662, 98268] avg=32768.0000 max=98268 max_rank=7 min=2 min_rank=1 imbalance_ratio=2.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[64592, 2, 2, 33726, 65536, 150, 65375, 32761] avg=32768.0000 max=65536 max_rank=4 min=2 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[32417, 32768, 32505, 32349, 33096, 32817, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32349 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=200892 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[33095, 31127, 33096, 33098, 33096, 32769, 32768, 33095] avg=32768.0000 max=33098 max_rank=3 min=31127 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=65203 max_load_reduction_pct=0.663299 moved_to_replicas=182169 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[32417, 32768, 32505, 32349, 33096, 32817, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32349 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=200892 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32442, 32281, 33096, 33096, 32276, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=32276 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=193309 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[32417, 32768, 32505, 32349, 33096, 32817, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32349 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=200892 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32442, 32281, 33096, 33096, 32276, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=32276 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=193309 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[64592, 2, 2, 33726, 65536, 150, 65375, 32761] avg=32768.0000 max=65536 max_rank=4 min=2 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[11, 2, 32505, 29061, 67818, 32817, 1662, 98268] avg=32768.0000 max=98268 max_rank=7 min=2 min_rank=1 imbalance_ratio=2.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[64592, 2, 2, 33726, 65536, 150, 65375, 32761] avg=32768.0000 max=65536 max_rank=4 min=2 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[11, 2, 32505, 29061, 67818, 32817, 1662, 98268] avg=32768.0000 max=98268 max_rank=7 min=2 min_rank=1 imbalance_ratio=2.998901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[64592, 2, 2, 33726, 65536, 150, 65375, 32761] avg=32768.0000 max=65536 max_rank=4 min=2 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[32759, 65531, 32768, 98212, 32758, 2, 17, 97] avg=32768.0000 max=98212 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32442, 32281, 33096, 33096, 32276, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=32276 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=193309 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[32417, 32768, 32505, 32349, 33096, 32817, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32349 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=200892 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[32759, 65531, 32768, 98212, 32758, 2, 17, 97] avg=32768.0000 max=98212 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32442, 32281, 33096, 33096, 32276, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=32276 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=193309 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[32417, 32768, 32505, 32349, 33096, 32817, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32349 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65172 max_load_reduction_pct=0.663207 moved_to_replicas=200892 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[64592, 2, 2, 33726, 65536, 150, 65375, 32761] avg=32768.0000 max=65536 max_rank=4 min=2 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32442, 32281, 33096, 33096, 32276, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=32276 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=193309 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[32759, 33096, 32768, 33096, 32758, 32768, 32367, 32532] avg=32768.0000 max=33096 max_rank=1 min=32367 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65116 max_load_reduction_pct=0.663015 moved_to_replicas=194897 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[32759, 33096, 32768, 33096, 32758, 32768, 32367, 32532] avg=32768.0000 max=33096 max_rank=1 min=32367 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65116 max_load_reduction_pct=0.663015 moved_to_replicas=194897 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[32759, 65531, 32768, 98212, 32758, 2, 17, 97] avg=32768.0000 max=98212 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32442, 32281, 33096, 33096, 32276, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=32276 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=193309 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[64592, 2, 2, 33726, 65536, 150, 65375, 32761] avg=32768.0000 max=65536 max_rank=4 min=2 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[32759, 65531, 32768, 98212, 32758, 2, 17, 97] avg=32768.0000 max=98212 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[64592, 2, 2, 33726, 65536, 150, 65375, 32761] avg=32768.0000 max=65536 max_rank=4 min=2 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[32759, 65531, 32768, 98212, 32758, 2, 17, 97] avg=32768.0000 max=98212 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[6, 32742, 90510, 32771, 32767, 7620, 32969, 32759] avg=32768.0000 max=90510 max_rank=2 min=6 min_rank=0 imbalance_ratio=2.762146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[32759, 33096, 32768, 33096, 32758, 32768, 32367, 32532] avg=32768.0000 max=33096 max_rank=1 min=32367 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65116 max_load_reduction_pct=0.663015 moved_to_replicas=194897 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32442, 32281, 33096, 33096, 32276, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=32276 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=193309 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[6, 32742, 90510, 32771, 32767, 7620, 32969, 32759] avg=32768.0000 max=90510 max_rank=2 min=6 min_rank=0 imbalance_ratio=2.762146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[32759, 33096, 32768, 33096, 32758, 32768, 32367, 32532] avg=32768.0000 max=33096 max_rank=1 min=32367 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65116 max_load_reduction_pct=0.663015 moved_to_replicas=194897 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32442, 32281, 33096, 33096, 32276, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=32276 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=193309 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[32759, 65531, 32768, 98212, 32758, 2, 17, 97] avg=32768.0000 max=98212 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[32759, 33096, 32768, 33096, 32758, 32768, 32367, 32532] avg=32768.0000 max=33096 max_rank=1 min=32367 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65116 max_load_reduction_pct=0.663015 moved_to_replicas=194897 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[32772, 32742, 33096, 32771, 32767, 32268, 32969, 32759] avg=32768.0000 max=33096 max_rank=2 min=32268 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57414 max_load_reduction_pct=0.634339 moved_to_replicas=113126 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[32772, 32742, 33096, 32771, 32767, 32268, 32969, 32759] avg=32768.0000 max=33096 max_rank=2 min=32268 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57414 max_load_reduction_pct=0.634339 moved_to_replicas=113126 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[6, 32742, 90510, 32771, 32767, 7620, 32969, 32759] avg=32768.0000 max=90510 max_rank=2 min=6 min_rank=0 imbalance_ratio=2.762146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[32759, 33096, 32768, 33096, 32758, 32768, 32367, 32532] avg=32768.0000 max=33096 max_rank=1 min=32367 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65116 max_load_reduction_pct=0.663015 moved_to_replicas=194897 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[32759, 65531, 32768, 98212, 32758, 2, 17, 97] avg=32768.0000 max=98212 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[6, 32742, 90510, 32771, 32767, 7620, 32969, 32759] avg=32768.0000 max=90510 max_rank=2 min=6 min_rank=0 imbalance_ratio=2.762146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[32759, 65531, 32768, 98212, 32758, 2, 17, 97] avg=32768.0000 max=98212 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[6, 32742, 90510, 32771, 32767, 7620, 32969, 32759] avg=32768.0000 max=90510 max_rank=2 min=6 min_rank=0 imbalance_ratio=2.762146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[29164, 41, 53411, 2, 65529, 32769, 32767, 48461] avg=32768.0000 max=65529 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[32772, 32742, 33096, 32771, 32767, 32268, 32969, 32759] avg=32768.0000 max=33096 max_rank=2 min=32268 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57414 max_load_reduction_pct=0.634339 moved_to_replicas=113126 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[29164, 41, 53411, 2, 65529, 32769, 32767, 48461] avg=32768.0000 max=65529 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[32759, 33096, 32768, 33096, 32758, 32768, 32367, 32532] avg=32768.0000 max=33096 max_rank=1 min=32367 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65116 max_load_reduction_pct=0.663015 moved_to_replicas=194897 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[32772, 32742, 33096, 32771, 32767, 32268, 32969, 32759] avg=32768.0000 max=33096 max_rank=2 min=32268 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57414 max_load_reduction_pct=0.634339 moved_to_replicas=113126 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[32759, 33096, 32768, 33096, 32758, 32768, 32367, 32532] avg=32768.0000 max=33096 max_rank=1 min=32367 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=65116 max_load_reduction_pct=0.663015 moved_to_replicas=194897 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[6, 32742, 90510, 32771, 32767, 7620, 32969, 32759] avg=32768.0000 max=90510 max_rank=2 min=6 min_rank=0 imbalance_ratio=2.762146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[32772, 32742, 33096, 32771, 32767, 32268, 32969, 32759] avg=32768.0000 max=33096 max_rank=2 min=32268 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57414 max_load_reduction_pct=0.634339 moved_to_replicas=113126 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[31789, 33094, 33097, 32435, 33096, 32769, 32767, 33097] avg=32768.0000 max=33097 max_rank=2 min=31789 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32432 max_load_reduction_pct=0.494926 moved_to_replicas=136145 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[31789, 33094, 33097, 32435, 33096, 32769, 32767, 33097] avg=32768.0000 max=33097 max_rank=2 min=31789 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32432 max_load_reduction_pct=0.494926 moved_to_replicas=136145 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[29164, 41, 53411, 2, 65529, 32769, 32767, 48461] avg=32768.0000 max=65529 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[32772, 32742, 33096, 32771, 32767, 32268, 32969, 32759] avg=32768.0000 max=33096 max_rank=2 min=32268 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57414 max_load_reduction_pct=0.634339 moved_to_replicas=113126 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[6, 32742, 90510, 32771, 32767, 7620, 32969, 32759] avg=32768.0000 max=90510 max_rank=2 min=6 min_rank=0 imbalance_ratio=2.762146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[29164, 41, 53411, 2, 65529, 32769, 32767, 48461] avg=32768.0000 max=65529 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[6, 32742, 90510, 32771, 32767, 7620, 32969, 32759] avg=32768.0000 max=90510 max_rank=2 min=6 min_rank=0 imbalance_ratio=2.762146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[29164, 41, 53411, 2, 65529, 32769, 32767, 48461] avg=32768.0000 max=65529 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[31789, 33094, 33097, 32435, 33096, 32769, 32767, 33097] avg=32768.0000 max=33097 max_rank=2 min=31789 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32432 max_load_reduction_pct=0.494926 moved_to_replicas=136145 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[1, 65463, 4, 32766, 65718, 32769, 69, 65354] avg=32768.0000 max=65718 max_rank=4 min=1 min_rank=0 imbalance_ratio=2.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[1, 65463, 4, 32766, 65718, 32769, 69, 65354] avg=32768.0000 max=65718 max_rank=4 min=1 min_rank=0 imbalance_ratio=2.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[32772, 32742, 33096, 32771, 32767, 32268, 32969, 32759] avg=32768.0000 max=33096 max_rank=2 min=32268 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57414 max_load_reduction_pct=0.634339 moved_to_replicas=113126 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[31789, 33094, 33097, 32435, 33096, 32769, 32767, 33097] avg=32768.0000 max=33097 max_rank=2 min=31789 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32432 max_load_reduction_pct=0.494926 moved_to_replicas=136145 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[32772, 32742, 33096, 32771, 32767, 32268, 32969, 32759] avg=32768.0000 max=33096 max_rank=2 min=32268 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57414 max_load_reduction_pct=0.634339 moved_to_replicas=113126 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[29164, 41, 53411, 2, 65529, 32769, 32767, 48461] avg=32768.0000 max=65529 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[31789, 33094, 33097, 32435, 33096, 32769, 32767, 33097] avg=32768.0000 max=33097 max_rank=2 min=31789 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32432 max_load_reduction_pct=0.494926 moved_to_replicas=136145 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[32623, 33096, 32371, 32766, 33096, 32769, 32327, 33096] avg=32768.0000 max=33096 max_rank=1 min=32327 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32622 max_load_reduction_pct=0.496394 moved_to_replicas=194458 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[32623, 33096, 32371, 32766, 33096, 32769, 32327, 33096] avg=32768.0000 max=33096 max_rank=1 min=32327 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32622 max_load_reduction_pct=0.496394 moved_to_replicas=194458 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[1, 65463, 4, 32766, 65718, 32769, 69, 65354] avg=32768.0000 max=65718 max_rank=4 min=1 min_rank=0 imbalance_ratio=2.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[31789, 33094, 33097, 32435, 33096, 32769, 32767, 33097] avg=32768.0000 max=33097 max_rank=2 min=31789 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32432 max_load_reduction_pct=0.494926 moved_to_replicas=136145 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[29164, 41, 53411, 2, 65529, 32769, 32767, 48461] avg=32768.0000 max=65529 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[1, 65463, 4, 32766, 65718, 32769, 69, 65354] avg=32768.0000 max=65718 max_rank=4 min=1 min_rank=0 imbalance_ratio=2.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[29164, 41, 53411, 2, 65529, 32769, 32767, 48461] avg=32768.0000 max=65529 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[32623, 33096, 32371, 32766, 33096, 32769, 32327, 33096] avg=32768.0000 max=33096 max_rank=1 min=32327 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32622 max_load_reduction_pct=0.496394 moved_to_replicas=194458 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[1, 65463, 4, 32766, 65718, 32769, 69, 65354] avg=32768.0000 max=65718 max_rank=4 min=1 min_rank=0 imbalance_ratio=2.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[98212, 32762, 32903, 32767, 1, 32730, 32768, 1] avg=32768.0000 max=98212 max_rank=0 min=1 min_rank=4 imbalance_ratio=2.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[98212, 32762, 32903, 32767, 1, 32730, 32768, 1] avg=32768.0000 max=98212 max_rank=0 min=1 min_rank=4 imbalance_ratio=2.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[31789, 33094, 33097, 32435, 33096, 32769, 32767, 33097] avg=32768.0000 max=33097 max_rank=2 min=31789 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32432 max_load_reduction_pct=0.494926 moved_to_replicas=136145 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[32623, 33096, 32371, 32766, 33096, 32769, 32327, 33096] avg=32768.0000 max=33096 max_rank=1 min=32327 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32622 max_load_reduction_pct=0.496394 moved_to_replicas=194458 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[1, 65463, 4, 32766, 65718, 32769, 69, 65354] avg=32768.0000 max=65718 max_rank=4 min=1 min_rank=0 imbalance_ratio=2.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[31789, 33094, 33097, 32435, 33096, 32769, 32767, 33097] avg=32768.0000 max=33097 max_rank=2 min=31789 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32432 max_load_reduction_pct=0.494926 moved_to_replicas=136145 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[32623, 33096, 32371, 32766, 33096, 32769, 32327, 33096] avg=32768.0000 max=33096 max_rank=1 min=32327 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32622 max_load_reduction_pct=0.496394 moved_to_replicas=194458 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[33096, 32762, 32903, 32767, 32757, 32730, 32768, 32361] avg=32768.0000 max=33096 max_rank=0 min=32361 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65116 max_load_reduction_pct=0.663015 moved_to_replicas=130210 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[33096, 32762, 32903, 32767, 32757, 32730, 32768, 32361] avg=32768.0000 max=33096 max_rank=0 min=32361 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65116 max_load_reduction_pct=0.663015 moved_to_replicas=130210 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[98212, 32762, 32903, 32767, 1, 32730, 32768, 1] avg=32768.0000 max=98212 max_rank=0 min=1 min_rank=4 imbalance_ratio=2.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[32623, 33096, 32371, 32766, 33096, 32769, 32327, 33096] avg=32768.0000 max=33096 max_rank=1 min=32327 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32622 max_load_reduction_pct=0.496394 moved_to_replicas=194458 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[1, 65463, 4, 32766, 65718, 32769, 69, 65354] avg=32768.0000 max=65718 max_rank=4 min=1 min_rank=0 imbalance_ratio=2.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[33096, 32762, 32903, 32767, 32757, 32730, 32768, 32361] avg=32768.0000 max=33096 max_rank=0 min=32361 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65116 max_load_reduction_pct=0.663015 moved_to_replicas=130210 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[98212, 32762, 32903, 32767, 1, 32730, 32768, 1] avg=32768.0000 max=98212 max_rank=0 min=1 min_rank=4 imbalance_ratio=2.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[1, 65463, 4, 32766, 65718, 32769, 69, 65354] avg=32768.0000 max=65718 max_rank=4 min=1 min_rank=0 imbalance_ratio=2.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[98212, 32762, 32903, 32767, 1, 32730, 32768, 1] avg=32768.0000 max=98212 max_rank=0 min=1 min_rank=4 imbalance_ratio=2.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[32623, 33096, 32371, 32766, 33096, 32769, 32327, 33096] avg=32768.0000 max=33096 max_rank=1 min=32327 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32622 max_load_reduction_pct=0.496394 moved_to_replicas=194458 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[98212, 32762, 32903, 32767, 1, 32730, 32768, 1] avg=32768.0000 max=98212 max_rank=0 min=1 min_rank=4 imbalance_ratio=2.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[33096, 32762, 32903, 32767, 32757, 32730, 32768, 32361] avg=32768.0000 max=33096 max_rank=0 min=32361 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65116 max_load_reduction_pct=0.663015 moved_to_replicas=130210 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[32623, 33096, 32371, 32766, 33096, 32769, 32327, 33096] avg=32768.0000 max=33096 max_rank=1 min=32327 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32622 max_load_reduction_pct=0.496394 moved_to_replicas=194458 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[33096, 32762, 32903, 32767, 32757, 32730, 32768, 32361] avg=32768.0000 max=33096 max_rank=0 min=32361 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65116 max_load_reduction_pct=0.663015 moved_to_replicas=130210 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[33096, 32762, 32903, 32767, 32757, 32730, 32768, 32361] avg=32768.0000 max=33096 max_rank=0 min=32361 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65116 max_load_reduction_pct=0.663015 moved_to_replicas=130210 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[98212, 32762, 32903, 32767, 1, 32730, 32768, 1] avg=32768.0000 max=98212 max_rank=0 min=1 min_rank=4 imbalance_ratio=2.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[98212, 32762, 32903, 32767, 1, 32730, 32768, 1] avg=32768.0000 max=98212 max_rank=0 min=1 min_rank=4 imbalance_ratio=2.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[33096, 32762, 32903, 32767, 32757, 32730, 32768, 32361] avg=32768.0000 max=33096 max_rank=0 min=32361 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65116 max_load_reduction_pct=0.663015 moved_to_replicas=130210 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[33096, 32762, 32903, 32767, 32757, 32730, 32768, 32361] avg=32768.0000 max=33096 max_rank=0 min=32361 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65116 max_load_reduction_pct=0.663015 moved_to_replicas=130210 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=32 total_expected_recv=1051441 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=51968 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=32 total_expected_recv=1047676 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=55936 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=32 total_expected_recv=1044775 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=60672 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=32 total_expected_recv=1052406 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=53632 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=32 total_expected_recv=1048264 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=55680 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=32 total_expected_recv=1047693 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=55424 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=32 total_expected_recv=1049957 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56192 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=32 total_expected_recv=1046396 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57216 peak_demand=33098 required_capacity=1.010071 + [2026-08-30 05:11:46] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 570.8 | throughput per GPU (TFLOP/s/GPU): 584.0 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.973547E+00 | loss scale: 1.0 | grad norm: 5.224 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=34816 expected_recv=66195 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32675, 33096, 31515, 33096, 32472, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32675, 33096, 31515, 33096, 32472, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32675, 33096, 31515, 33096, 32472, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32675, 33096, 31515, 33096, 32472, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=34816 expected_recv=65167 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32675, 33096, 31515, 33096, 32472, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=34816 expected_recv=64983 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32675, 33096, 31515, 33096, 32472, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32675, 33096, 31515, 33096, 32472, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=34816 expected_recv=66195 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=34816 expected_recv=63175 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32675, 33096, 31515, 33096, 32472, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=34816 expected_recv=65167 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=34816 expected_recv=64983 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=34816 expected_recv=63175 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=2 received_num_tokens=69632 valid_num_tokens=64640 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=34816 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[33096, 33093, 33097, 33098, 32765, 32763, 33096, 31136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=34816 expected_recv=65197 expected_overflow=0 exact_recv_per_rank=[33096, 33093, 33097, 33098, 32765, 32763, 33096, 31136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=34816 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33096, 33093, 33097, 33098, 32765, 32763, 33096, 31136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=34816 expected_recv=65514 expected_overflow=0 exact_recv_per_rank=[33096, 33093, 33097, 33098, 32765, 32763, 33096, 31136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33093, 33097, 33098, 32765, 32763, 33096, 31136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=34816 expected_recv=66190 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=34816 expected_recv=65197 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33093, 33097, 33098, 32765, 32763, 33096, 31136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=34816 expected_recv=66194 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=34816 expected_recv=65514 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=34816 expected_recv=63284 expected_overflow=0 exact_recv_per_rank=[33096, 33093, 33097, 33098, 32765, 32763, 33096, 31136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=34816 expected_recv=65524 expected_overflow=0 exact_recv_per_rank=[33096, 33093, 33097, 33098, 32765, 32763, 33096, 31136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=34816 expected_recv=63284 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=34816 expected_recv=65524 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=64256 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=34816 expected_recv=63927 expected_overflow=0 exact_recv_per_rank=[32047, 32795, 33096, 32763, 32361, 32889, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=34816 expected_recv=65985 expected_overflow=0 exact_recv_per_rank=[32047, 32795, 33096, 32763, 32361, 32889, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=34816 expected_recv=65565 expected_overflow=0 exact_recv_per_rank=[32047, 32795, 33096, 32763, 32361, 32889, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32047, 32795, 33096, 32763, 32361, 32889, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=34816 expected_recv=63927 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=34816 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[32047, 32795, 33096, 32763, 32361, 32889, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32047, 32795, 33096, 32763, 32361, 32889, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=34816 expected_recv=65985 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=34816 expected_recv=65565 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=64640 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=34816 expected_recv=65532 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32047, 32795, 33096, 32763, 32361, 32889, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=34816 expected_recv=64702 expected_overflow=0 exact_recv_per_rank=[32047, 32795, 33096, 32763, 32361, 32889, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=34816 expected_recv=64702 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=34816 expected_recv=64839 expected_overflow=0 exact_recv_per_rank=[32530, 32578, 32535, 32736, 33096, 32477, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=34816 expected_recv=65225 expected_overflow=0 exact_recv_per_rank=[32530, 32578, 32535, 32736, 33096, 32477, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=34816 expected_recv=64839 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=34816 expected_recv=65225 expected_overflow=0 exact_recv_per_rank=[32530, 32578, 32535, 32736, 33096, 32477, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=34816 expected_recv=64929 expected_overflow=0 exact_recv_per_rank=[32530, 32578, 32535, 32736, 33096, 32477, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=34816 expected_recv=65225 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=34816 expected_recv=65494 expected_overflow=0 exact_recv_per_rank=[32530, 32578, 32535, 32736, 33096, 32477, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=34816 expected_recv=65225 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32530, 32578, 32535, 32736, 33096, 32477, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=34816 expected_recv=64929 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=34816 expected_recv=65494 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32530, 32578, 32535, 32736, 33096, 32477, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32530, 32578, 32535, 32736, 33096, 32477, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=34816 expected_recv=64800 expected_overflow=0 exact_recv_per_rank=[32439, 33096, 33097, 33096, 32045, 33095, 32763, 32513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=34816 expected_recv=66033 expected_overflow=0 exact_recv_per_rank=[32439, 33096, 33097, 33096, 32045, 33095, 32763, 32513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=34816 expected_recv=64800 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32439, 33096, 33097, 33096, 32045, 33095, 32763, 32513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=34816 expected_recv=66033 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32439, 33096, 33097, 33096, 32045, 33095, 32763, 32513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32439, 33096, 33097, 33096, 32045, 33095, 32763, 32513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=34816 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[32439, 33096, 33097, 33096, 32045, 33095, 32763, 32513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=34816 expected_recv=64619 expected_overflow=0 exact_recv_per_rank=[32439, 33096, 33097, 33096, 32045, 33095, 32763, 32513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=34816 expected_recv=64732 expected_overflow=0 exact_recv_per_rank=[32439, 33096, 33097, 33096, 32045, 33095, 32763, 32513] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=34816 expected_recv=65527 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=34816 expected_recv=64619 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=34816 expected_recv=64732 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32249, 33096, 31983, 32766, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=34816 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32249, 33096, 31983, 32766, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32249, 33096, 31983, 32766, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=34816 expected_recv=65533 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=34816 expected_recv=64541 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32249, 33096, 31983, 32766, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32249, 33096, 31983, 32766, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32249, 33096, 31983, 32766, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=34816 expected_recv=64541 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=34816 expected_recv=63944 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32249, 33096, 31983, 32766, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=34816 expected_recv=65503 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32249, 33096, 31983, 32766, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=34816 expected_recv=63944 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=34816 expected_recv=65503 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=34816 expected_recv=65524 expected_overflow=0 exact_recv_per_rank=[32763, 32666, 32466, 33096, 33096, 32844, 32451, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=34816 expected_recv=65524 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=34816 expected_recv=65877 expected_overflow=0 exact_recv_per_rank=[32763, 32666, 32466, 33096, 33096, 32844, 32451, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=34816 expected_recv=65039 expected_overflow=0 exact_recv_per_rank=[32763, 32666, 32466, 33096, 33096, 32844, 32451, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=34816 expected_recv=65877 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=34816 expected_recv=65062 expected_overflow=0 exact_recv_per_rank=[32763, 32666, 32466, 33096, 33096, 32844, 32451, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=34816 expected_recv=65039 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32763, 32666, 32466, 33096, 33096, 32844, 32451, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32763, 32666, 32466, 33096, 33096, 32844, 32451, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=34816 expected_recv=65062 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=34816 expected_recv=64876 expected_overflow=0 exact_recv_per_rank=[32763, 32666, 32466, 33096, 33096, 32844, 32451, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=34816 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[32763, 32666, 32466, 33096, 33096, 32844, 32451, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32486, 33096, 33095, 33096, 31729, 33097, 32449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=34816 expected_recv=64876 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=34816 expected_recv=65526 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=34816 expected_recv=63680 expected_overflow=0 exact_recv_per_rank=[33096, 32486, 33096, 33095, 33096, 31729, 33097, 32449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=34816 expected_recv=64754 expected_overflow=0 exact_recv_per_rank=[33096, 32486, 33096, 33095, 33096, 31729, 33097, 32449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=34816 expected_recv=63680 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=64512 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=34816 expected_recv=64754 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32486, 33096, 33095, 33096, 31729, 33097, 32449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32486, 33096, 33095, 33096, 31729, 33097, 32449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32486, 33096, 33095, 33096, 31729, 33097, 32449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32486, 33096, 33095, 33096, 31729, 33097, 32449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=34816 expected_recv=64894 expected_overflow=0 exact_recv_per_rank=[33096, 32486, 33096, 33095, 33096, 31729, 33097, 32449] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32603, 33096, 32678, 33096, 31704, 32773, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=34816 expected_recv=64894 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=34816 expected_recv=64800 expected_overflow=0 exact_recv_per_rank=[33096, 32603, 33096, 32678, 33096, 31704, 32773, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=34816 expected_recv=64800 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=34816 expected_recv=65159 expected_overflow=0 exact_recv_per_rank=[33096, 32603, 33096, 32678, 33096, 31704, 32773, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32603, 33096, 32678, 33096, 31704, 32773, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=34816 expected_recv=64817 expected_overflow=0 exact_recv_per_rank=[33096, 32603, 33096, 32678, 33096, 31704, 32773, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=34816 expected_recv=65159 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=34816 expected_recv=65340 expected_overflow=0 exact_recv_per_rank=[33096, 32603, 33096, 32678, 33096, 31704, 32773, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=34816 expected_recv=65594 expected_overflow=0 exact_recv_per_rank=[33096, 32603, 33096, 32678, 33096, 31704, 32773, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=34816 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33096, 32603, 33096, 32678, 33096, 31704, 32773, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=34816 expected_recv=64817 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=34816 expected_recv=62911 expected_overflow=0 exact_recv_per_rank=[31858, 33096, 33097, 33096, 33095, 32845, 31961, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=34816 expected_recv=65340 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=34816 expected_recv=65594 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=34816 expected_recv=66194 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=34816 expected_recv=62911 actual_valid_recv=63616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=34816 expected_recv=65806 expected_overflow=0 exact_recv_per_rank=[31858, 33096, 33097, 33096, 33095, 32845, 31961, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=63616 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=34816 expected_recv=65806 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=34816 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31858, 33096, 33097, 33096, 33095, 32845, 31961, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=34816 expected_recv=66195 expected_overflow=0 exact_recv_per_rank=[31858, 33096, 33097, 33096, 33095, 32845, 31961, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=34816 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[31858, 33096, 33097, 33096, 33095, 32845, 31961, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31858, 33096, 33097, 33096, 33095, 32845, 31961, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=34816 expected_recv=66193 actual_valid_recv=67328 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=34816 expected_recv=66195 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=34816 expected_recv=66190 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=34816 expected_recv=64610 expected_overflow=0 exact_recv_per_rank=[31858, 33096, 33097, 33096, 33095, 32845, 31961, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=34816 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[31858, 33096, 33097, 33096, 33095, 32845, 31961, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=34816 expected_recv=65645 expected_overflow=0 exact_recv_per_rank=[32920, 33096, 32458, 33096, 33096, 31752, 32966, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=67328 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=34816 expected_recv=64610 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=34816 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=34816 expected_recv=65645 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=34816 expected_recv=63623 expected_overflow=0 exact_recv_per_rank=[32920, 33096, 32458, 33096, 33096, 31752, 32966, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=65280 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=34816 expected_recv=63623 actual_valid_recv=64000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=64000 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32920, 33096, 32458, 33096, 33096, 31752, 32966, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32920, 33096, 32458, 33096, 33096, 31752, 32966, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=34816 expected_recv=64918 expected_overflow=0 exact_recv_per_rank=[32920, 33096, 32458, 33096, 33096, 31752, 32966, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=34816 expected_recv=64918 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32920, 33096, 32458, 33096, 33096, 31752, 32966, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=34816 expected_recv=65513 expected_overflow=0 exact_recv_per_rank=[32920, 33096, 32458, 33096, 33096, 31752, 32966, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=34816 expected_recv=66013 expected_overflow=0 exact_recv_per_rank=[32920, 33096, 32458, 33096, 33096, 31752, 32966, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=34816 expected_recv=65407 expected_overflow=0 exact_recv_per_rank=[32741, 33096, 32770, 33096, 32424, 32770, 32927, 32320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=34816 expected_recv=65513 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=34816 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[32741, 33096, 32770, 33096, 32424, 32770, 32927, 32320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=34816 expected_recv=66013 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=34816 expected_recv=65407 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=67072 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=34816 expected_recv=65545 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32741, 33096, 32770, 33096, 32424, 32770, 32927, 32320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=34816 expected_recv=64800 expected_overflow=0 exact_recv_per_rank=[32741, 33096, 32770, 33096, 32424, 32770, 32927, 32320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=34816 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[32741, 33096, 32770, 33096, 32424, 32770, 32927, 32320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=34816 expected_recv=64800 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=34816 expected_recv=65542 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32741, 33096, 32770, 33096, 32424, 32770, 32927, 32320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=34816 expected_recv=65355 expected_overflow=0 exact_recv_per_rank=[32741, 33096, 32770, 33096, 32424, 32770, 32927, 32320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=34816 expected_recv=65255 expected_overflow=0 exact_recv_per_rank=[32741, 33096, 32770, 33096, 32424, 32770, 32927, 32320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=34816 expected_recv=65358 expected_overflow=0 exact_recv_per_rank=[32500, 31767, 33096, 33096, 32753, 33096, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32500, 31767, 33096, 33096, 32753, 33096, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=34816 expected_recv=65355 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=34816 expected_recv=65255 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=34816 expected_recv=65358 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=34816 expected_recv=63282 expected_overflow=0 exact_recv_per_rank=[32500, 31767, 33096, 33096, 32753, 33096, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=34816 expected_recv=65493 expected_overflow=0 exact_recv_per_rank=[32500, 31767, 33096, 33096, 32753, 33096, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32500, 31767, 33096, 33096, 32753, 33096, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=34816 expected_recv=63282 actual_valid_recv=64128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=34816 expected_recv=65493 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32500, 31767, 33096, 33096, 32753, 33096, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=64128 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32500, 31767, 33096, 33096, 32753, 33096, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=34816 expected_recv=65387 expected_overflow=0 exact_recv_per_rank=[32500, 31767, 33096, 33096, 32753, 33096, 33096, 32740] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32392, 33096, 32425, 33096, 32769, 32768, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=34816 expected_recv=65572 expected_overflow=0 exact_recv_per_rank=[33096, 32392, 33096, 32425, 33096, 32769, 32768, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=34816 expected_recv=65387 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=34816 expected_recv=65572 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=12 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66816 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32392, 33096, 32425, 33096, 32769, 32768, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=34816 expected_recv=65403 expected_overflow=0 exact_recv_per_rank=[33096, 32392, 33096, 32425, 33096, 32769, 32768, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32392, 33096, 32425, 33096, 32769, 32768, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=34816 expected_recv=65403 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66432 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=34816 expected_recv=64794 expected_overflow=0 exact_recv_per_rank=[33096, 32392, 33096, 32425, 33096, 32769, 32768, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=34816 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[33096, 32392, 33096, 32425, 33096, 32769, 32768, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=34816 expected_recv=64413 expected_overflow=0 exact_recv_per_rank=[33096, 32392, 33096, 32425, 33096, 32769, 32768, 32502] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=34816 expected_recv=64821 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32280, 32443, 33096, 32770, 32596, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=34816 expected_recv=64794 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=34816 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32280, 32443, 33096, 32770, 32596, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=34816 expected_recv=65530 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=34816 expected_recv=64413 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=34816 expected_recv=64821 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65408 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=34816 expected_recv=65550 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=13 calls=2 received_num_tokens=69632 valid_num_tokens=65024 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65664 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32280, 32443, 33096, 32770, 32596, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66048 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32280, 32443, 33096, 32770, 32596, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=34816 expected_recv=65067 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32280, 32443, 33096, 32770, 32596, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=34816 expected_recv=65067 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66176 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=34816 expected_recv=64900 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32280, 32443, 33096, 32770, 32596, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=34816 expected_recv=65374 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32280, 32443, 33096, 32770, 32596, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32280, 32443, 33096, 32770, 32596, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=34816 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32614, 32864, 32769, 32751, 32527, 32768, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=34816 expected_recv=64652 expected_overflow=0 exact_recv_per_rank=[33096, 32614, 32864, 32769, 32751, 32527, 32768, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=34816 expected_recv=64900 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=34816 expected_recv=65374 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=34816 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=34816 expected_recv=64652 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65536 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=34816 expected_recv=65484 expected_overflow=0 exact_recv_per_rank=[33096, 32614, 32864, 32769, 32751, 32527, 32768, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=14 calls=2 received_num_tokens=69632 valid_num_tokens=66560 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66944 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65152 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=34816 expected_recv=65710 expected_overflow=0 exact_recv_per_rank=[33096, 32614, 32864, 32769, 32751, 32527, 32768, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=34816 expected_recv=65484 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=34816 expected_recv=65674 expected_overflow=0 exact_recv_per_rank=[33096, 32614, 32864, 32769, 32751, 32527, 32768, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=34816 expected_recv=65710 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=34816 expected_recv=65674 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66688 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=66304 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=34816 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[33096, 32614, 32864, 32769, 32751, 32527, 32768, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=34816 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[33096, 32614, 32864, 32769, 32751, 32527, 32768, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=34816 expected_recv=65509 expected_overflow=0 exact_recv_per_rank=[33096, 32614, 32864, 32769, 32751, 32527, 32768, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=34816 expected_recv=65532 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=34816 expected_recv=65535 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[38979, 34362, 31409, 34849, 4289, 36940, 32472, 48844] avg=32768.0000 max=48844 max_rank=7 min=4289 min_rank=4 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[38979, 34362, 31409, 34849, 4289, 36940, 32472, 48844] avg=32768.0000 max=48844 max_rank=7 min=4289 min_rank=4 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=34816 expected_recv=65509 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65792 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=15 calls=2 received_num_tokens=69632 valid_num_tokens=65920 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[38979, 34362, 31409, 34849, 4289, 36940, 32472, 48844] avg=32768.0000 max=48844 max_rank=7 min=4289 min_rank=4 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[38979, 34362, 31409, 34849, 4289, 36940, 32472, 48844] avg=32768.0000 max=48844 max_rank=7 min=4289 min_rank=4 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[38979, 34362, 31409, 34849, 4289, 36940, 32472, 48844] avg=32768.0000 max=48844 max_rank=7 min=4289 min_rank=4 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[33097, 33096, 32675, 33096, 31515, 33096, 32472, 33097] avg=32768.0000 max=33097 max_rank=0 min=31515 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=58273 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[33097, 33096, 32675, 33096, 31515, 33096, 32472, 33097] avg=32768.0000 max=33097 max_rank=0 min=31515 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=58273 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[33097, 33096, 32675, 33096, 31515, 33096, 32472, 33097] avg=32768.0000 max=33097 max_rank=0 min=31515 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=58273 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[38979, 34362, 31409, 34849, 4289, 36940, 32472, 48844] avg=32768.0000 max=48844 max_rank=7 min=4289 min_rank=4 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[38979, 34362, 31409, 34849, 4289, 36940, 32472, 48844] avg=32768.0000 max=48844 max_rank=7 min=4289 min_rank=4 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[33097, 33096, 32675, 33096, 31515, 33096, 32472, 33097] avg=32768.0000 max=33097 max_rank=0 min=31515 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=58273 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[38979, 34362, 31409, 34849, 4289, 36940, 32472, 48844] avg=32768.0000 max=48844 max_rank=7 min=4289 min_rank=4 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[33097, 33096, 32675, 33096, 31515, 33096, 32472, 33097] avg=32768.0000 max=33097 max_rank=0 min=31515 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=58273 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[4172, 11463, 111095, 39865, 32765, 6, 33577, 29201] avg=32768.0000 max=111095 max_rank=2 min=6 min_rank=5 imbalance_ratio=3.390350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[4172, 11463, 111095, 39865, 32765, 6, 33577, 29201] avg=32768.0000 max=111095 max_rank=2 min=6 min_rank=5 imbalance_ratio=3.390350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[33097, 33096, 32675, 33096, 31515, 33096, 32472, 33097] avg=32768.0000 max=33097 max_rank=0 min=31515 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=58273 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[4172, 11463, 111095, 39865, 32765, 6, 33577, 29201] avg=32768.0000 max=111095 max_rank=2 min=6 min_rank=5 imbalance_ratio=3.390350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[33097, 33096, 32675, 33096, 31515, 33096, 32472, 33097] avg=32768.0000 max=33097 max_rank=0 min=31515 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=58273 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[33097, 33096, 32675, 33096, 31515, 33096, 32472, 33097] avg=32768.0000 max=33097 max_rank=0 min=31515 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=58273 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33093, 33097, 33098, 32765, 32763, 33096, 31136] avg=32768.0000 max=33098 max_rank=3 min=31136 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=77997 max_load_reduction_pct=0.702075 moved_to_replicas=170967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[4172, 11463, 111095, 39865, 32765, 6, 33577, 29201] avg=32768.0000 max=111095 max_rank=2 min=6 min_rank=5 imbalance_ratio=3.390350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33093, 33097, 33098, 32765, 32763, 33096, 31136] avg=32768.0000 max=33098 max_rank=3 min=31136 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=77997 max_load_reduction_pct=0.702075 moved_to_replicas=170967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[4172, 11463, 111095, 39865, 32765, 6, 33577, 29201] avg=32768.0000 max=111095 max_rank=2 min=6 min_rank=5 imbalance_ratio=3.390350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33093, 33097, 33098, 32765, 32763, 33096, 31136] avg=32768.0000 max=33098 max_rank=3 min=31136 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=77997 max_load_reduction_pct=0.702075 moved_to_replicas=170967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[4172, 11463, 111095, 39865, 32765, 6, 33577, 29201] avg=32768.0000 max=111095 max_rank=2 min=6 min_rank=5 imbalance_ratio=3.390350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[4172, 11463, 111095, 39865, 32765, 6, 33577, 29201] avg=32768.0000 max=111095 max_rank=2 min=6 min_rank=5 imbalance_ratio=3.390350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33093, 33097, 33098, 32765, 32763, 33096, 31136] avg=32768.0000 max=33098 max_rank=3 min=31136 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=77997 max_load_reduction_pct=0.702075 moved_to_replicas=170967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[4172, 11463, 111095, 39865, 32765, 6, 33577, 29201] avg=32768.0000 max=111095 max_rank=2 min=6 min_rank=5 imbalance_ratio=3.390350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33093, 33097, 33098, 32765, 32763, 33096, 31136] avg=32768.0000 max=33098 max_rank=3 min=31136 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=77997 max_load_reduction_pct=0.702075 moved_to_replicas=170967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[32047, 32795, 38580, 32763, 13, 1225, 59277, 65444] avg=32768.0000 max=65444 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[32047, 32795, 38580, 32763, 13, 1225, 59277, 65444] avg=32768.0000 max=65444 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[32047, 32795, 38580, 32763, 13, 1225, 59277, 65444] avg=32768.0000 max=65444 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33093, 33097, 33098, 32765, 32763, 33096, 31136] avg=32768.0000 max=33098 max_rank=3 min=31136 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=77997 max_load_reduction_pct=0.702075 moved_to_replicas=170967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33093, 33097, 33098, 32765, 32763, 33096, 31136] avg=32768.0000 max=33098 max_rank=3 min=31136 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=77997 max_load_reduction_pct=0.702075 moved_to_replicas=170967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33093, 33097, 33098, 32765, 32763, 33096, 31136] avg=32768.0000 max=33098 max_rank=3 min=31136 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=77997 max_load_reduction_pct=0.702075 moved_to_replicas=170967 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[32047, 32795, 33096, 32763, 32361, 32889, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=32047 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32347 max_load_reduction_pct=0.494270 moved_to_replicas=127860 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[32047, 32795, 33096, 32763, 32361, 32889, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=32047 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32347 max_load_reduction_pct=0.494270 moved_to_replicas=127860 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[32047, 32795, 38580, 32763, 13, 1225, 59277, 65444] avg=32768.0000 max=65444 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[32047, 32795, 33096, 32763, 32361, 32889, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=32047 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32347 max_load_reduction_pct=0.494270 moved_to_replicas=127860 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[32047, 32795, 38580, 32763, 13, 1225, 59277, 65444] avg=32768.0000 max=65444 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[32047, 32795, 38580, 32763, 13, 1225, 59277, 65444] avg=32768.0000 max=65444 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[32047, 32795, 38580, 32763, 13, 1225, 59277, 65444] avg=32768.0000 max=65444 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[32047, 32795, 33096, 32763, 32361, 32889, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=32047 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32347 max_load_reduction_pct=0.494270 moved_to_replicas=127860 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[32047, 32795, 38580, 32763, 13, 1225, 59277, 65444] avg=32768.0000 max=65444 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.997192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[32047, 32795, 33096, 32763, 32361, 32889, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=32047 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32347 max_load_reduction_pct=0.494270 moved_to_replicas=127860 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[8, 32578, 32390, 6, 97942, 32477, 33241, 33502] avg=32768.0000 max=97942 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.988953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[8, 32578, 32390, 6, 97942, 32477, 33241, 33502] avg=32768.0000 max=97942 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.988953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[8, 32578, 32390, 6, 97942, 32477, 33241, 33502] avg=32768.0000 max=97942 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.988953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[32047, 32795, 33096, 32763, 32361, 32889, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=32047 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32347 max_load_reduction_pct=0.494270 moved_to_replicas=127860 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[32047, 32795, 33096, 32763, 32361, 32889, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=32047 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32347 max_load_reduction_pct=0.494270 moved_to_replicas=127860 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[32047, 32795, 33096, 32763, 32361, 32889, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=32047 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32347 max_load_reduction_pct=0.494270 moved_to_replicas=127860 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[32530, 32578, 32535, 32736, 33096, 32477, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32477 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64846 max_load_reduction_pct=0.662086 moved_to_replicas=130775 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[32530, 32578, 32535, 32736, 33096, 32477, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32477 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64846 max_load_reduction_pct=0.662086 moved_to_replicas=130775 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[32530, 32578, 32535, 32736, 33096, 32477, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32477 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64846 max_load_reduction_pct=0.662086 moved_to_replicas=130775 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[8, 32578, 32390, 6, 97942, 32477, 33241, 33502] avg=32768.0000 max=97942 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.988953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[8, 32578, 32390, 6, 97942, 32477, 33241, 33502] avg=32768.0000 max=97942 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.988953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[8, 32578, 32390, 6, 97942, 32477, 33241, 33502] avg=32768.0000 max=97942 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.988953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[8, 32578, 32390, 6, 97942, 32477, 33241, 33502] avg=32768.0000 max=97942 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.988953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[8, 32578, 32390, 6, 97942, 32477, 33241, 33502] avg=32768.0000 max=97942 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.988953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[84, 65451, 46283, 44395, 32045, 20552, 32763, 20571] avg=32768.0000 max=65451 max_rank=1 min=84 min_rank=0 imbalance_ratio=1.997406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[32530, 32578, 32535, 32736, 33096, 32477, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32477 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64846 max_load_reduction_pct=0.662086 moved_to_replicas=130775 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[32530, 32578, 32535, 32736, 33096, 32477, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32477 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64846 max_load_reduction_pct=0.662086 moved_to_replicas=130775 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[84, 65451, 46283, 44395, 32045, 20552, 32763, 20571] avg=32768.0000 max=65451 max_rank=1 min=84 min_rank=0 imbalance_ratio=1.997406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[84, 65451, 46283, 44395, 32045, 20552, 32763, 20571] avg=32768.0000 max=65451 max_rank=1 min=84 min_rank=0 imbalance_ratio=1.997406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[32530, 32578, 32535, 32736, 33096, 32477, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32477 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64846 max_load_reduction_pct=0.662086 moved_to_replicas=130775 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[32530, 32578, 32535, 32736, 33096, 32477, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32477 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64846 max_load_reduction_pct=0.662086 moved_to_replicas=130775 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[32530, 32578, 32535, 32736, 33096, 32477, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32477 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64846 max_load_reduction_pct=0.662086 moved_to_replicas=130775 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[32439, 33096, 33097, 33096, 32045, 33095, 32763, 32513] avg=32768.0000 max=33097 max_rank=2 min=32045 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32354 max_load_reduction_pct=0.494324 moved_to_replicas=111116 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[32439, 33096, 33097, 33096, 32045, 33095, 32763, 32513] avg=32768.0000 max=33097 max_rank=2 min=32045 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32354 max_load_reduction_pct=0.494324 moved_to_replicas=111116 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[32439, 33096, 33097, 33096, 32045, 33095, 32763, 32513] avg=32768.0000 max=33097 max_rank=2 min=32045 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32354 max_load_reduction_pct=0.494324 moved_to_replicas=111116 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[84, 65451, 46283, 44395, 32045, 20552, 32763, 20571] avg=32768.0000 max=65451 max_rank=1 min=84 min_rank=0 imbalance_ratio=1.997406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[84, 65451, 46283, 44395, 32045, 20552, 32763, 20571] avg=32768.0000 max=65451 max_rank=1 min=84 min_rank=0 imbalance_ratio=1.997406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[84, 65451, 46283, 44395, 32045, 20552, 32763, 20571] avg=32768.0000 max=65451 max_rank=1 min=84 min_rank=0 imbalance_ratio=1.997406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[84, 65451, 46283, 44395, 32045, 20552, 32763, 20571] avg=32768.0000 max=65451 max_rank=1 min=84 min_rank=0 imbalance_ratio=1.997406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[84, 65451, 46283, 44395, 32045, 20552, 32763, 20571] avg=32768.0000 max=65451 max_rank=1 min=84 min_rank=0 imbalance_ratio=1.997406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[66083, 85690, 32249, 43779, 19, 32766, 1554, 4] avg=32768.0000 max=85690 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.615051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[32439, 33096, 33097, 33096, 32045, 33095, 32763, 32513] avg=32768.0000 max=33097 max_rank=2 min=32045 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32354 max_load_reduction_pct=0.494324 moved_to_replicas=111116 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[32439, 33096, 33097, 33096, 32045, 33095, 32763, 32513] avg=32768.0000 max=33097 max_rank=2 min=32045 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32354 max_load_reduction_pct=0.494324 moved_to_replicas=111116 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[66083, 85690, 32249, 43779, 19, 32766, 1554, 4] avg=32768.0000 max=85690 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.615051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[66083, 85690, 32249, 43779, 19, 32766, 1554, 4] avg=32768.0000 max=85690 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.615051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[32439, 33096, 33097, 33096, 32045, 33095, 32763, 32513] avg=32768.0000 max=33097 max_rank=2 min=32045 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32354 max_load_reduction_pct=0.494324 moved_to_replicas=111116 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[32439, 33096, 33097, 33096, 32045, 33095, 32763, 32513] avg=32768.0000 max=33097 max_rank=2 min=32045 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32354 max_load_reduction_pct=0.494324 moved_to_replicas=111116 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[32439, 33096, 33097, 33096, 32045, 33095, 32763, 32513] avg=32768.0000 max=33097 max_rank=2 min=32045 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32354 max_load_reduction_pct=0.494324 moved_to_replicas=111116 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[33096, 33096, 32249, 33096, 31983, 32766, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=31983 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52594 max_load_reduction_pct=0.613771 moved_to_replicas=191745 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[33096, 33096, 32249, 33096, 31983, 32766, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=31983 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52594 max_load_reduction_pct=0.613771 moved_to_replicas=191745 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[33096, 33096, 32249, 33096, 31983, 32766, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=31983 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52594 max_load_reduction_pct=0.613771 moved_to_replicas=191745 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[66083, 85690, 32249, 43779, 19, 32766, 1554, 4] avg=32768.0000 max=85690 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.615051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[66083, 85690, 32249, 43779, 19, 32766, 1554, 4] avg=32768.0000 max=85690 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.615051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[66083, 85690, 32249, 43779, 19, 32766, 1554, 4] avg=32768.0000 max=85690 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.615051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[33096, 33096, 32249, 33096, 31983, 32766, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=31983 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52594 max_load_reduction_pct=0.613771 moved_to_replicas=191745 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[33096, 33096, 32249, 33096, 31983, 32766, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=31983 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52594 max_load_reduction_pct=0.613771 moved_to_replicas=191745 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[66083, 85690, 32249, 43779, 19, 32766, 1554, 4] avg=32768.0000 max=85690 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.615051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[32763, 32666, 42, 65541, 65520, 32844, 6, 32762] avg=32768.0000 max=65541 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[32763, 32666, 42, 65541, 65520, 32844, 6, 32762] avg=32768.0000 max=65541 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[66083, 85690, 32249, 43779, 19, 32766, 1554, 4] avg=32768.0000 max=85690 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.615051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[32763, 32666, 42, 65541, 65520, 32844, 6, 32762] avg=32768.0000 max=65541 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[33096, 33096, 32249, 33096, 31983, 32766, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=31983 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52594 max_load_reduction_pct=0.613771 moved_to_replicas=191745 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[32763, 32666, 32466, 33096, 33096, 32844, 32451, 32762] avg=32768.0000 max=33096 max_rank=3 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129641 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[32763, 32666, 32466, 33096, 33096, 32844, 32451, 32762] avg=32768.0000 max=33096 max_rank=3 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129641 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[33096, 33096, 32249, 33096, 31983, 32766, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=31983 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52594 max_load_reduction_pct=0.613771 moved_to_replicas=191745 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[32763, 32666, 32466, 33096, 33096, 32844, 32451, 32762] avg=32768.0000 max=33096 max_rank=3 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129641 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[32763, 32666, 42, 65541, 65520, 32844, 6, 32762] avg=32768.0000 max=65541 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[33096, 33096, 32249, 33096, 31983, 32766, 33096, 32762] avg=32768.0000 max=33096 max_rank=0 min=31983 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52594 max_load_reduction_pct=0.613771 moved_to_replicas=191745 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[32763, 32666, 42, 65541, 65520, 32844, 6, 32762] avg=32768.0000 max=65541 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[32763, 32666, 42, 65541, 65520, 32844, 6, 32762] avg=32768.0000 max=65541 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[32763, 32666, 32466, 33096, 33096, 32844, 32451, 32762] avg=32768.0000 max=33096 max_rank=3 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129641 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[64084, 32486, 63968, 8928, 65532, 86, 27047, 13] avg=32768.0000 max=65532 max_rank=4 min=13 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[64084, 32486, 63968, 8928, 65532, 86, 27047, 13] avg=32768.0000 max=65532 max_rank=4 min=13 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[32763, 32666, 32466, 33096, 33096, 32844, 32451, 32762] avg=32768.0000 max=33096 max_rank=3 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129641 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[64084, 32486, 63968, 8928, 65532, 86, 27047, 13] avg=32768.0000 max=65532 max_rank=4 min=13 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[32763, 32666, 32466, 33096, 33096, 32844, 32451, 32762] avg=32768.0000 max=33096 max_rank=3 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129641 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[32763, 32666, 42, 65541, 65520, 32844, 6, 32762] avg=32768.0000 max=65541 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[33096, 32486, 33096, 33095, 33096, 31729, 33097, 32449] avg=32768.0000 max=33097 max_rank=6 min=31729 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=32435 max_load_reduction_pct=0.494949 moved_to_replicas=188976 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[33096, 32486, 33096, 33095, 33096, 31729, 33097, 32449] avg=32768.0000 max=33097 max_rank=6 min=31729 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=32435 max_load_reduction_pct=0.494949 moved_to_replicas=188976 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[32763, 32666, 42, 65541, 65520, 32844, 6, 32762] avg=32768.0000 max=65541 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.000153 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[33096, 32486, 33096, 33095, 33096, 31729, 33097, 32449] avg=32768.0000 max=33097 max_rank=6 min=31729 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=32435 max_load_reduction_pct=0.494949 moved_to_replicas=188976 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[64084, 32486, 63968, 8928, 65532, 86, 27047, 13] avg=32768.0000 max=65532 max_rank=4 min=13 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[64084, 32486, 63968, 8928, 65532, 86, 27047, 13] avg=32768.0000 max=65532 max_rank=4 min=13 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[32763, 32666, 32466, 33096, 33096, 32844, 32451, 32762] avg=32768.0000 max=33096 max_rank=3 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129641 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[64084, 32486, 63968, 8928, 65532, 86, 27047, 13] avg=32768.0000 max=65532 max_rank=4 min=13 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[32763, 32666, 32466, 33096, 33096, 32844, 32451, 32762] avg=32768.0000 max=33096 max_rank=3 min=32451 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32445 max_load_reduction_pct=0.495034 moved_to_replicas=129641 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[33096, 32486, 33096, 33095, 33096, 31729, 33097, 32449] avg=32768.0000 max=33097 max_rank=6 min=31729 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=32435 max_load_reduction_pct=0.494949 moved_to_replicas=188976 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[34552, 169, 98294, 32678, 33242, 19930, 9, 43270] avg=32768.0000 max=98294 max_rank=2 min=9 min_rank=6 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[34552, 169, 98294, 32678, 33242, 19930, 9, 43270] avg=32768.0000 max=98294 max_rank=2 min=9 min_rank=6 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[33096, 32486, 33096, 33095, 33096, 31729, 33097, 32449] avg=32768.0000 max=33097 max_rank=6 min=31729 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=32435 max_load_reduction_pct=0.494949 moved_to_replicas=188976 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[34552, 169, 98294, 32678, 33242, 19930, 9, 43270] avg=32768.0000 max=98294 max_rank=2 min=9 min_rank=6 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[33096, 32486, 33096, 33095, 33096, 31729, 33097, 32449] avg=32768.0000 max=33097 max_rank=6 min=31729 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=32435 max_load_reduction_pct=0.494949 moved_to_replicas=188976 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[64084, 32486, 63968, 8928, 65532, 86, 27047, 13] avg=32768.0000 max=65532 max_rank=4 min=13 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[33096, 32603, 33096, 32678, 33096, 31704, 32773, 33098] avg=32768.0000 max=33098 max_rank=7 min=31704 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=65196 max_load_reduction_pct=0.663275 moved_to_replicas=155327 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[33096, 32603, 33096, 32678, 33096, 31704, 32773, 33098] avg=32768.0000 max=33098 max_rank=7 min=31704 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=65196 max_load_reduction_pct=0.663275 moved_to_replicas=155327 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[33096, 32603, 33096, 32678, 33096, 31704, 32773, 33098] avg=32768.0000 max=33098 max_rank=7 min=31704 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=65196 max_load_reduction_pct=0.663275 moved_to_replicas=155327 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[64084, 32486, 63968, 8928, 65532, 86, 27047, 13] avg=32768.0000 max=65532 max_rank=4 min=13 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[34552, 169, 98294, 32678, 33242, 19930, 9, 43270] avg=32768.0000 max=98294 max_rank=2 min=9 min_rank=6 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[34552, 169, 98294, 32678, 33242, 19930, 9, 43270] avg=32768.0000 max=98294 max_rank=2 min=9 min_rank=6 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[33096, 32486, 33096, 33095, 33096, 31729, 33097, 32449] avg=32768.0000 max=33097 max_rank=6 min=31729 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=32435 max_load_reduction_pct=0.494949 moved_to_replicas=188976 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[34552, 169, 98294, 32678, 33242, 19930, 9, 43270] avg=32768.0000 max=98294 max_rank=2 min=9 min_rank=6 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[33096, 32486, 33096, 33095, 33096, 31729, 33097, 32449] avg=32768.0000 max=33097 max_rank=6 min=31729 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=32435 max_load_reduction_pct=0.494949 moved_to_replicas=188976 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[42, 140, 58013, 2403, 72649, 32845, 26127, 69925] avg=32768.0000 max=72649 max_rank=4 min=42 min_rank=0 imbalance_ratio=2.217072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[33096, 32603, 33096, 32678, 33096, 31704, 32773, 33098] avg=32768.0000 max=33098 max_rank=7 min=31704 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=65196 max_load_reduction_pct=0.663275 moved_to_replicas=155327 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[42, 140, 58013, 2403, 72649, 32845, 26127, 69925] avg=32768.0000 max=72649 max_rank=4 min=42 min_rank=0 imbalance_ratio=2.217072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[33096, 32603, 33096, 32678, 33096, 31704, 32773, 33098] avg=32768.0000 max=33098 max_rank=7 min=31704 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=65196 max_load_reduction_pct=0.663275 moved_to_replicas=155327 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[42, 140, 58013, 2403, 72649, 32845, 26127, 69925] avg=32768.0000 max=72649 max_rank=4 min=42 min_rank=0 imbalance_ratio=2.217072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[33096, 32603, 33096, 32678, 33096, 31704, 32773, 33098] avg=32768.0000 max=33098 max_rank=7 min=31704 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=65196 max_load_reduction_pct=0.663275 moved_to_replicas=155327 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[31858, 33096, 33097, 33096, 33095, 32845, 31961, 33096] avg=32768.0000 max=33097 max_rank=2 min=31858 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=39552 max_load_reduction_pct=0.544426 moved_to_replicas=200421 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[34552, 169, 98294, 32678, 33242, 19930, 9, 43270] avg=32768.0000 max=98294 max_rank=2 min=9 min_rank=6 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[31858, 33096, 33097, 33096, 33095, 32845, 31961, 33096] avg=32768.0000 max=33097 max_rank=2 min=31858 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=39552 max_load_reduction_pct=0.544426 moved_to_replicas=200421 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[31858, 33096, 33097, 33096, 33095, 32845, 31961, 33096] avg=32768.0000 max=33097 max_rank=2 min=31858 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=39552 max_load_reduction_pct=0.544426 moved_to_replicas=200421 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[34552, 169, 98294, 32678, 33242, 19930, 9, 43270] avg=32768.0000 max=98294 max_rank=2 min=9 min_rank=6 imbalance_ratio=2.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[42, 140, 58013, 2403, 72649, 32845, 26127, 69925] avg=32768.0000 max=72649 max_rank=4 min=42 min_rank=0 imbalance_ratio=2.217072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[42, 140, 58013, 2403, 72649, 32845, 26127, 69925] avg=32768.0000 max=72649 max_rank=4 min=42 min_rank=0 imbalance_ratio=2.217072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[33096, 32603, 33096, 32678, 33096, 31704, 32773, 33098] avg=32768.0000 max=33098 max_rank=7 min=31704 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=65196 max_load_reduction_pct=0.663275 moved_to_replicas=155327 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[42, 140, 58013, 2403, 72649, 32845, 26127, 69925] avg=32768.0000 max=72649 max_rank=4 min=42 min_rank=0 imbalance_ratio=2.217072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[32920, 9939, 20, 65454, 65534, 22551, 32966, 32760] avg=32768.0000 max=65534 max_rank=4 min=20 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[33096, 32603, 33096, 32678, 33096, 31704, 32773, 33098] avg=32768.0000 max=33098 max_rank=7 min=31704 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=65196 max_load_reduction_pct=0.663275 moved_to_replicas=155327 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[32920, 9939, 20, 65454, 65534, 22551, 32966, 32760] avg=32768.0000 max=65534 max_rank=4 min=20 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[31858, 33096, 33097, 33096, 33095, 32845, 31961, 33096] avg=32768.0000 max=33097 max_rank=2 min=31858 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=39552 max_load_reduction_pct=0.544426 moved_to_replicas=200421 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[31858, 33096, 33097, 33096, 33095, 32845, 31961, 33096] avg=32768.0000 max=33097 max_rank=2 min=31858 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=39552 max_load_reduction_pct=0.544426 moved_to_replicas=200421 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[32920, 9939, 20, 65454, 65534, 22551, 32966, 32760] avg=32768.0000 max=65534 max_rank=4 min=20 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[31858, 33096, 33097, 33096, 33095, 32845, 31961, 33096] avg=32768.0000 max=33097 max_rank=2 min=31858 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=39552 max_load_reduction_pct=0.544426 moved_to_replicas=200421 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[32920, 33096, 32458, 33096, 33096, 31752, 32966, 32760] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=129562 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[32920, 33096, 32458, 33096, 33096, 31752, 32966, 32760] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=129562 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[42, 140, 58013, 2403, 72649, 32845, 26127, 69925] avg=32768.0000 max=72649 max_rank=4 min=42 min_rank=0 imbalance_ratio=2.217072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[32920, 33096, 32458, 33096, 33096, 31752, 32966, 32760] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=129562 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[42, 140, 58013, 2403, 72649, 32845, 26127, 69925] avg=32768.0000 max=72649 max_rank=4 min=42 min_rank=0 imbalance_ratio=2.217072 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[32920, 9939, 20, 65454, 65534, 22551, 32966, 32760] avg=32768.0000 max=65534 max_rank=4 min=20 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[32920, 9939, 20, 65454, 65534, 22551, 32966, 32760] avg=32768.0000 max=65534 max_rank=4 min=20 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[31858, 33096, 33097, 33096, 33095, 32845, 31961, 33096] avg=32768.0000 max=33097 max_rank=2 min=31858 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=39552 max_load_reduction_pct=0.544426 moved_to_replicas=200421 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[32920, 9939, 20, 65454, 65534, 22551, 32966, 32760] avg=32768.0000 max=65534 max_rank=4 min=20 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[32741, 66201, 32770, 98098, 32082, 6, 164, 82] avg=32768.0000 max=98098 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.993713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[32741, 66201, 32770, 98098, 32082, 6, 164, 82] avg=32768.0000 max=98098 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.993713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[31858, 33096, 33097, 33096, 33095, 32845, 31961, 33096] avg=32768.0000 max=33097 max_rank=2 min=31858 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=39552 max_load_reduction_pct=0.544426 moved_to_replicas=200421 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[32920, 33096, 32458, 33096, 33096, 31752, 32966, 32760] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=129562 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[32741, 66201, 32770, 98098, 32082, 6, 164, 82] avg=32768.0000 max=98098 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.993713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[32920, 33096, 32458, 33096, 33096, 31752, 32966, 32760] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=129562 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[32920, 33096, 32458, 33096, 33096, 31752, 32966, 32760] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=129562 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[32741, 33096, 32770, 33096, 32424, 32770, 32927, 32320] avg=32768.0000 max=33096 max_rank=1 min=32320 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65002 max_load_reduction_pct=0.662623 moved_to_replicas=195762 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[32741, 33096, 32770, 33096, 32424, 32770, 32927, 32320] avg=32768.0000 max=33096 max_rank=1 min=32320 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65002 max_load_reduction_pct=0.662623 moved_to_replicas=195762 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[32920, 9939, 20, 65454, 65534, 22551, 32966, 32760] avg=32768.0000 max=65534 max_rank=4 min=20 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[32741, 33096, 32770, 33096, 32424, 32770, 32927, 32320] avg=32768.0000 max=33096 max_rank=1 min=32320 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65002 max_load_reduction_pct=0.662623 moved_to_replicas=195762 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[32920, 9939, 20, 65454, 65534, 22551, 32966, 32760] avg=32768.0000 max=65534 max_rank=4 min=20 min_rank=2 imbalance_ratio=1.999939 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[32741, 66201, 32770, 98098, 32082, 6, 164, 82] avg=32768.0000 max=98098 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.993713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[32741, 66201, 32770, 98098, 32082, 6, 164, 82] avg=32768.0000 max=98098 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.993713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[32920, 33096, 32458, 33096, 33096, 31752, 32966, 32760] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=129562 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[32741, 66201, 32770, 98098, 32082, 6, 164, 82] avg=32768.0000 max=98098 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.993713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[21, 30674, 65575, 33187, 32753, 3586, 63608, 32740] avg=32768.0000 max=65575 max_rank=2 min=21 min_rank=0 imbalance_ratio=2.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[21, 30674, 65575, 33187, 32753, 3586, 63608, 32740] avg=32768.0000 max=65575 max_rank=2 min=21 min_rank=0 imbalance_ratio=2.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[32920, 33096, 32458, 33096, 33096, 31752, 32966, 32760] avg=32768.0000 max=33096 max_rank=1 min=31752 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32438 max_load_reduction_pct=0.494980 moved_to_replicas=129562 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[32741, 33096, 32770, 33096, 32424, 32770, 32927, 32320] avg=32768.0000 max=33096 max_rank=1 min=32320 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65002 max_load_reduction_pct=0.662623 moved_to_replicas=195762 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[32741, 33096, 32770, 33096, 32424, 32770, 32927, 32320] avg=32768.0000 max=33096 max_rank=1 min=32320 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65002 max_load_reduction_pct=0.662623 moved_to_replicas=195762 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[21, 30674, 65575, 33187, 32753, 3586, 63608, 32740] avg=32768.0000 max=65575 max_rank=2 min=21 min_rank=0 imbalance_ratio=2.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[32741, 33096, 32770, 33096, 32424, 32770, 32927, 32320] avg=32768.0000 max=33096 max_rank=1 min=32320 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65002 max_load_reduction_pct=0.662623 moved_to_replicas=195762 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[32500, 31767, 33096, 33096, 32753, 33096, 33096, 32740] avg=32768.0000 max=33096 max_rank=2 min=31767 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=126769 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[32500, 31767, 33096, 33096, 32753, 33096, 33096, 32740] avg=32768.0000 max=33096 max_rank=2 min=31767 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=126769 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[32741, 66201, 32770, 98098, 32082, 6, 164, 82] avg=32768.0000 max=98098 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.993713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[32500, 31767, 33096, 33096, 32753, 33096, 33096, 32740] avg=32768.0000 max=33096 max_rank=2 min=31767 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=126769 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[32741, 66201, 32770, 98098, 32082, 6, 164, 82] avg=32768.0000 max=98098 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.993713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[21, 30674, 65575, 33187, 32753, 3586, 63608, 32740] avg=32768.0000 max=65575 max_rank=2 min=21 min_rank=0 imbalance_ratio=2.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[21, 30674, 65575, 33187, 32753, 3586, 63608, 32740] avg=32768.0000 max=65575 max_rank=2 min=21 min_rank=0 imbalance_ratio=2.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[32741, 33096, 32770, 33096, 32424, 32770, 32927, 32320] avg=32768.0000 max=33096 max_rank=1 min=32320 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65002 max_load_reduction_pct=0.662623 moved_to_replicas=195762 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[21, 30674, 65575, 33187, 32753, 3586, 63608, 32740] avg=32768.0000 max=65575 max_rank=2 min=21 min_rank=0 imbalance_ratio=2.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[64893, 222, 33469, 6, 65515, 32769, 32768, 32502] avg=32768.0000 max=65515 max_rank=4 min=6 min_rank=3 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[64893, 222, 33469, 6, 65515, 32769, 32768, 32502] avg=32768.0000 max=65515 max_rank=4 min=6 min_rank=3 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[32741, 33096, 32770, 33096, 32424, 32770, 32927, 32320] avg=32768.0000 max=33096 max_rank=1 min=32320 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65002 max_load_reduction_pct=0.662623 moved_to_replicas=195762 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[32500, 31767, 33096, 33096, 32753, 33096, 33096, 32740] avg=32768.0000 max=33096 max_rank=2 min=31767 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=126769 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[32500, 31767, 33096, 33096, 32753, 33096, 33096, 32740] avg=32768.0000 max=33096 max_rank=2 min=31767 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=126769 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[64893, 222, 33469, 6, 65515, 32769, 32768, 32502] avg=32768.0000 max=65515 max_rank=4 min=6 min_rank=3 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[32500, 31767, 33096, 33096, 32753, 33096, 33096, 32740] avg=32768.0000 max=33096 max_rank=2 min=31767 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=126769 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32392, 33096, 32425, 33096, 32769, 32768, 32502] avg=32768.0000 max=33096 max_rank=0 min=32392 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=129446 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32392, 33096, 32425, 33096, 32769, 32768, 32502] avg=32768.0000 max=33096 max_rank=0 min=32392 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=129446 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[21, 30674, 65575, 33187, 32753, 3586, 63608, 32740] avg=32768.0000 max=65575 max_rank=2 min=21 min_rank=0 imbalance_ratio=2.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32392, 33096, 32425, 33096, 32769, 32768, 32502] avg=32768.0000 max=33096 max_rank=0 min=32392 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=129446 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[21, 30674, 65575, 33187, 32753, 3586, 63608, 32740] avg=32768.0000 max=65575 max_rank=2 min=21 min_rank=0 imbalance_ratio=2.001190 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[64893, 222, 33469, 6, 65515, 32769, 32768, 32502] avg=32768.0000 max=65515 max_rank=4 min=6 min_rank=3 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[64893, 222, 33469, 6, 65515, 32769, 32768, 32502] avg=32768.0000 max=65515 max_rank=4 min=6 min_rank=3 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[64893, 222, 33469, 6, 65515, 32769, 32768, 32502] avg=32768.0000 max=65515 max_rank=4 min=6 min_rank=3 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[6, 64573, 9, 415, 98128, 32770, 1119, 65124] avg=32768.0000 max=98128 max_rank=4 min=6 min_rank=0 imbalance_ratio=2.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[6, 64573, 9, 415, 98128, 32770, 1119, 65124] avg=32768.0000 max=98128 max_rank=4 min=6 min_rank=0 imbalance_ratio=2.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[32500, 31767, 33096, 33096, 32753, 33096, 33096, 32740] avg=32768.0000 max=33096 max_rank=2 min=31767 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=126769 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[32500, 31767, 33096, 33096, 32753, 33096, 33096, 32740] avg=32768.0000 max=33096 max_rank=2 min=31767 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32479 max_load_reduction_pct=0.495295 moved_to_replicas=126769 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32392, 33096, 32425, 33096, 32769, 32768, 32502] avg=32768.0000 max=33096 max_rank=0 min=32392 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=129446 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[6, 64573, 9, 415, 98128, 32770, 1119, 65124] avg=32768.0000 max=98128 max_rank=4 min=6 min_rank=0 imbalance_ratio=2.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32392, 33096, 32425, 33096, 32769, 32768, 32502] avg=32768.0000 max=33096 max_rank=0 min=32392 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=129446 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32392, 33096, 32425, 33096, 32769, 32768, 32502] avg=32768.0000 max=33096 max_rank=0 min=32392 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=129446 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[32767, 33096, 32280, 32443, 33096, 32770, 32596, 33096] avg=32768.0000 max=33096 max_rank=1 min=32280 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65032 max_load_reduction_pct=0.662726 moved_to_replicas=255549 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[32767, 33096, 32280, 32443, 33096, 32770, 32596, 33096] avg=32768.0000 max=33096 max_rank=1 min=32280 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65032 max_load_reduction_pct=0.662726 moved_to_replicas=255549 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[64893, 222, 33469, 6, 65515, 32769, 32768, 32502] avg=32768.0000 max=65515 max_rank=4 min=6 min_rank=3 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[32767, 33096, 32280, 32443, 33096, 32770, 32596, 33096] avg=32768.0000 max=33096 max_rank=1 min=32280 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65032 max_load_reduction_pct=0.662726 moved_to_replicas=255549 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[64893, 222, 33469, 6, 65515, 32769, 32768, 32502] avg=32768.0000 max=65515 max_rank=4 min=6 min_rank=3 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[6, 64573, 9, 415, 98128, 32770, 1119, 65124] avg=32768.0000 max=98128 max_rank=4 min=6 min_rank=0 imbalance_ratio=2.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[130496, 715, 32864, 32769, 3, 32527, 32768, 2] avg=32768.0000 max=130496 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.982422 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[130496, 715, 32864, 32769, 3, 32527, 32768, 2] avg=32768.0000 max=130496 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.982422 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[6, 64573, 9, 415, 98128, 32770, 1119, 65124] avg=32768.0000 max=98128 max_rank=4 min=6 min_rank=0 imbalance_ratio=2.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32392, 33096, 32425, 33096, 32769, 32768, 32502] avg=32768.0000 max=33096 max_rank=0 min=32392 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=129446 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[6, 64573, 9, 415, 98128, 32770, 1119, 65124] avg=32768.0000 max=98128 max_rank=4 min=6 min_rank=0 imbalance_ratio=2.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32392, 33096, 32425, 33096, 32769, 32768, 32502] avg=32768.0000 max=33096 max_rank=0 min=32392 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=129446 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[32767, 33096, 32280, 32443, 33096, 32770, 32596, 33096] avg=32768.0000 max=33096 max_rank=1 min=32280 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65032 max_load_reduction_pct=0.662726 moved_to_replicas=255549 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[33096, 32614, 32864, 32769, 32751, 32527, 32768, 32755] avg=32768.0000 max=33096 max_rank=0 min=32527 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97400 max_load_reduction_pct=0.746383 moved_to_replicas=194779 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[33096, 32614, 32864, 32769, 32751, 32527, 32768, 32755] avg=32768.0000 max=33096 max_rank=0 min=32527 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97400 max_load_reduction_pct=0.746383 moved_to_replicas=194779 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[130496, 715, 32864, 32769, 3, 32527, 32768, 2] avg=32768.0000 max=130496 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.982422 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[32767, 33096, 32280, 32443, 33096, 32770, 32596, 33096] avg=32768.0000 max=33096 max_rank=1 min=32280 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65032 max_load_reduction_pct=0.662726 moved_to_replicas=255549 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[32767, 33096, 32280, 32443, 33096, 32770, 32596, 33096] avg=32768.0000 max=33096 max_rank=1 min=32280 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65032 max_load_reduction_pct=0.662726 moved_to_replicas=255549 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[6, 64573, 9, 415, 98128, 32770, 1119, 65124] avg=32768.0000 max=98128 max_rank=4 min=6 min_rank=0 imbalance_ratio=2.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[33096, 32614, 32864, 32769, 32751, 32527, 32768, 32755] avg=32768.0000 max=33096 max_rank=0 min=32527 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97400 max_load_reduction_pct=0.746383 moved_to_replicas=194779 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[6, 64573, 9, 415, 98128, 32770, 1119, 65124] avg=32768.0000 max=98128 max_rank=4 min=6 min_rank=0 imbalance_ratio=2.994629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[130496, 715, 32864, 32769, 3, 32527, 32768, 2] avg=32768.0000 max=130496 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.982422 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[32767, 33096, 32280, 32443, 33096, 32770, 32596, 33096] avg=32768.0000 max=33096 max_rank=1 min=32280 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65032 max_load_reduction_pct=0.662726 moved_to_replicas=255549 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[130496, 715, 32864, 32769, 3, 32527, 32768, 2] avg=32768.0000 max=130496 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.982422 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[130496, 715, 32864, 32769, 3, 32527, 32768, 2] avg=32768.0000 max=130496 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.982422 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[32767, 33096, 32280, 32443, 33096, 32770, 32596, 33096] avg=32768.0000 max=33096 max_rank=1 min=32280 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65032 max_load_reduction_pct=0.662726 moved_to_replicas=255549 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[33096, 32614, 32864, 32769, 32751, 32527, 32768, 32755] avg=32768.0000 max=33096 max_rank=0 min=32527 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97400 max_load_reduction_pct=0.746383 moved_to_replicas=194779 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[33096, 32614, 32864, 32769, 32751, 32527, 32768, 32755] avg=32768.0000 max=33096 max_rank=0 min=32527 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97400 max_load_reduction_pct=0.746383 moved_to_replicas=194779 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[33096, 32614, 32864, 32769, 32751, 32527, 32768, 32755] avg=32768.0000 max=33096 max_rank=0 min=32527 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97400 max_load_reduction_pct=0.746383 moved_to_replicas=194779 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[130496, 715, 32864, 32769, 3, 32527, 32768, 2] avg=32768.0000 max=130496 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.982422 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[130496, 715, 32864, 32769, 3, 32527, 32768, 2] avg=32768.0000 max=130496 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.982422 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[33096, 32614, 32864, 32769, 32751, 32527, 32768, 32755] avg=32768.0000 max=33096 max_rank=0 min=32527 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97400 max_load_reduction_pct=0.746383 moved_to_replicas=194779 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[33096, 32614, 32864, 32769, 32751, 32527, 32768, 32755] avg=32768.0000 max=33096 max_rank=0 min=32527 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=97400 max_load_reduction_pct=0.746383 moved_to_replicas=194779 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=32 total_expected_recv=1045900 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57088 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=32 total_expected_recv=1045779 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=58240 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=32 total_expected_recv=1046577 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56192 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=32 total_expected_recv=1050442 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=51968 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=32 total_expected_recv=1053513 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=50304 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=32 total_expected_recv=1048679 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=52992 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=32 total_expected_recv=1047269 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57856 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=32 total_expected_recv=1050449 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=54400 peak_demand=33097 required_capacity=1.010040 + [2026-08-30 05:11:46] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 569.1 | throughput per GPU (TFLOP/s/GPU): 585.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.845339E+00 | loss scale: 1.0 | grad norm: 5.022 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +[after training is done] datetime: 2026-08-30 05:11:46 +[rank0]:[W830 05:11:47.164019234 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W830 05:11:49.491585886 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W830 05:11:49.574347734 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W830 05:11:49.574481476 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W830 05:11:49.575346904 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W830 05:11:49.575548195 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W830 05:11:49.575790824 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W830 05:11:49.206812431 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 05:11:53.056000 488791 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 05:11:53.057000 488791 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 488884 closing signal SIGTERM +W0830 05:11:53.057000 488791 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 488885 closing signal SIGTERM +W0830 05:11:53.058000 488791 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 488886 closing signal SIGTERM +W0830 05:11:53.058000 488791 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 488887 closing signal SIGTERM +W0830 05:11:53.059000 488791 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 488888 closing signal SIGTERM +W0830 05:11:53.059000 488791 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 488889 closing signal SIGTERM +W0830 05:11:53.060000 488791 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 488890 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 488791 got signal: 15 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..69cdfa40927c3fdd8b432e0ac09bf1b96a10fd8f --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 2 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 2 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1895 +1, 1895 +2, 1895 +3, 1895 +4, 1895 +5, 1895 +6, 1895 +7, 1895 +0, 2387 +1, 2389 +2, 2403 +3, 2423 +4, 2407 +5, 2397 +6, 2409 +7, 2425 +0, 42337 +1, 42333 +2, 42333 +3, 42333 +4, 42333 +5, 42333 +6, 42333 +7, 42333 +0, 42743 +1, 42733 +2, 42733 +3, 42733 +4, 42733 +5, 42733 +6, 42733 +7, 42733 +0, 44049 +1, 44041 +2, 44041 +3, 44041 +4, 44041 +5, 44041 +6, 44041 +7, 44041 +0, 44049 +1, 44041 +2, 44041 +3, 44041 +4, 44041 +5, 44041 +6, 44041 +7, 44041 +0, 44369 +1, 44361 +2, 44361 +3, 44361 +4, 44361 +5, 44361 +6, 44361 +7, 44361 +0, 46011 +1, 46003 +2, 46003 +3, 46003 +4, 46003 +5, 46003 +6, 46003 +7, 46003 +0, 56661 +1, 56653 +2, 56653 +3, 56653 +4, 56653 +5, 56653 +6, 56653 +7, 56653 +0, 72849 +1, 72841 +2, 72841 +3, 72841 +4, 72841 +5, 72841 +6, 72841 +7, 72841 +0, 73409 +1, 73401 +2, 73401 +3, 73401 +4, 73401 +5, 73401 +6, 73401 +7, 73401 +0, 73409 +1, 73401 +2, 73401 +3, 73401 +4, 73401 +5, 73401 +6, 73401 +7, 73401 +0, 85139 +1, 85189 +2, 85205 +3, 85219 +4, 85131 +5, 85171 +6, 85233 +7, 85199 +0, 86239 +1, 86301 +2, 86301 +3, 86301 +4, 86231 +5, 86301 +6, 86301 +7, 86301 +0, 87255 +1, 87315 +2, 87331 +3, 87345 +4, 87263 +5, 87333 +6, 87317 +7, 87297 +0, 110343 +1, 110333 +2, 110333 +3, 110333 +4, 110319 +5, 110333 +6, 110333 +7, 110333 +0, 110343 +1, 110333 +2, 110333 +3, 110333 +4, 110319 +5, 110333 +6, 110333 +7, 110333 +0, 110343 +1, 110333 +2, 110333 +3, 110333 +4, 110319 +5, 110333 +6, 110333 +7, 110333 +0, 109427 +1, 109417 +2, 109481 +3, 110063 +4, 109723 +5, 109417 +6, 109417 +7, 109417 +0, 105631 +1, 105815 +2, 105941 +3, 105815 +4, 105475 +5, 105621 +6, 105815 +7, 105489 +0, 103563 +1, 104135 +2, 103873 +3, 104135 +4, 103795 +5, 103553 +6, 104135 +7, 103809 +0, 65 +1, 104134 +2, 324 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1/run.env b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..e55ee5b1afec6a72e955415b850259bc9c68bd7f --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1/run.env @@ -0,0 +1,70 @@ +backend=ultraep +config=G2 +family=tierA +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=3584 +num_experts=32 +topk=8 +experts_per_rank=4 +expert_params=44040192 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=32768 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=4 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=1024 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1/train.log b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..0f6cf09059f337094c1e45ef901573bc6b9d89cf --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1/train.log @@ -0,0 +1,8956 @@ +[05:09:28] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + G2 / ultraep (slots/rank=4 policy=matched) + E=32 K=8 H'=3584 m=4 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=static cap=1.25 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0830 05:09:29.856000 485280 torch/distributed/run.py:766] +W0830 05:09:29.856000 485280 torch/distributed/run.py:766] ***************************************** +W0830 05:09:29.856000 485280 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 05:09:29.856000 485280 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 3584 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 4 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 32 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 3 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 77 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank4]:[W830 05:09:48.327629902 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 05:09:48.329178990 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 05:09:48.329188854 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 05:09:48.329201991 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 05:09:48.329242639 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 05:09:48.329357406 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 05:09:48.329365138 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.036 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 05:09:48.519585861 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 4.041 seconds +time to initialize megatron (seconds): 7.332 +[after megatron is initialized] datetime: 2026-08-30 05:09:54 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 6722555904 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (42086400 elements, 42086400 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_proj.weight +Params for bucket 3 (42082304 elements, 42082304 padded size): + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_qkv.weight +Params for bucket 4 (42082304 elements, 42082304 padded size): + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight +Params for bucket 5 (42082304 elements, 42082304 padded size): + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight +Params for bucket 6 (42082304 elements, 42082304 padded size): + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_proj.weight +Params for bucket 7 (42082304 elements, 42082304 padded size): + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight +Params for bucket 8 (42082304 elements, 42082304 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight +Params for bucket 9 (42082304 elements, 42082304 padded size): + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_qkv.weight +Params for bucket 10 (42082304 elements, 42082304 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.pre_mlp_layernorm.weight +Params for bucket 11 (42082304 elements, 42082304 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.pre_mlp_layernorm.weight +Params for bucket 12 (42082304 elements, 42082304 padded size): + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight +Params for bucket 13 (42082304 elements, 42082304 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.pre_mlp_layernorm.weight +Params for bucket 14 (42082304 elements, 42082304 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.pre_mlp_layernorm.weight +Params for bucket 15 (42082304 elements, 42082304 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.pre_mlp_layernorm.weight +Params for bucket 16 (42082304 elements, 42082304 padded size): + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight +Params for bucket 17 (42082304 elements, 42082304 padded size): + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_proj.weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 56 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 +Params for bucket 3 (58720256 elements, 58720256 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 +Params for bucket 5 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 +Params for bucket 6 (58720256 elements, 58720256 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 +Params for bucket 7 (58720256 elements, 58720256 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 +Params for bucket 9 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 +Params for bucket 10 (58720256 elements, 58720256 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 +Params for bucket 13 (58720256 elements, 58720256 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 +Params for bucket 14 (58720256 elements, 58720256 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 17 (58720256 elements, 58720256 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 +Params for bucket 18 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 +Params for bucket 20 (58720256 elements, 58720256 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 +Params for bucket 21 (58720256 elements, 58720256 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 +Params for bucket 23 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 +Params for bucket 24 (58720256 elements, 58720256 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 +Params for bucket 27 (58720256 elements, 58720256 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 +Params for bucket 28 (58720256 elements, 58720256 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 29 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 30 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 31 (58720256 elements, 58720256 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 +Params for bucket 34 (58720256 elements, 58720256 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 +Params for bucket 35 (58720256 elements, 58720256 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 +Params for bucket 36 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 +Params for bucket 37 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 38 (58720256 elements, 58720256 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 +Params for bucket 41 (58720256 elements, 58720256 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 +Params for bucket 42 (58720256 elements, 58720256 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 43 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 +Params for bucket 44 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 +Params for bucket 45 (58720256 elements, 58720256 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 +Params for bucket 47 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 +Params for bucket 48 (58720256 elements, 58720256 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 +Params for bucket 49 (58720256 elements, 58720256 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 50 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 52 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +Params for bucket 53 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 +Params for bucket 54 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 +Params for bucket 55 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 +Params for bucket 56 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 05:09:57 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 192 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 05:09:57 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3355.55, 3359.36) + train/valid/test-data-iterators-setup ..........: (79.62, 92.92) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-30 05:09:57 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=40960 expected_recv=66152 expected_overflow=0 exact_recv_per_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=40960 expected_recv=64263 expected_overflow=0 exact_recv_per_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=40960 expected_recv=64282 expected_overflow=0 exact_recv_per_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=40960 expected_recv=64825 expected_overflow=0 exact_recv_per_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=2 budget=40960 expected_recv=66152 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=2 budget=40960 expected_recv=64263 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=2 budget=40960 expected_recv=64282 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=2 budget=40960 expected_recv=64825 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=40960 expected_recv=63808 expected_overflow=0 exact_recv_per_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=40960 expected_recv=64596 expected_overflow=0 exact_recv_per_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=2 budget=40960 expected_recv=63808 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=2 budget=40960 expected_recv=64596 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=40960 expected_recv=65733 expected_overflow=0 exact_recv_per_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=2 budget=40960 expected_recv=65733 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65964 expected_overflow=0 exact_recv_per_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65385 expected_overflow=0 exact_recv_per_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65307 expected_overflow=0 exact_recv_per_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65964 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65385 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=40960 expected_recv=64374 expected_overflow=0 exact_recv_per_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=2 budget=40960 expected_recv=65307 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=2 budget=40960 expected_recv=64374 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=40960 expected_recv=64537 expected_overflow=0 exact_recv_per_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=40960 expected_recv=64042 expected_overflow=0 exact_recv_per_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65695 expected_overflow=0 exact_recv_per_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=2 budget=40960 expected_recv=64537 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65719 expected_overflow=0 exact_recv_per_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=2 budget=40960 expected_recv=64042 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65695 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65719 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=40960 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=40960 expected_recv=65137 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=40960 expected_recv=64213 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=40960 expected_recv=64731 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=2 budget=40960 expected_recv=66190 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=2 budget=40960 expected_recv=65137 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=2 budget=40960 expected_recv=64213 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=2 budget=40960 expected_recv=64731 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=40960 expected_recv=63567 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=2 budget=40960 expected_recv=63567 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=40960 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=2 budget=40960 expected_recv=63568 actual_valid_recv=64000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=40960 expected_recv=65381 expected_overflow=0 exact_recv_per_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=64000 slack_slots=17920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=40960 expected_recv=66037 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=2 budget=40960 expected_recv=65381 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=2 budget=40960 expected_recv=66037 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=40960 expected_recv=64569 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=40960 expected_recv=63722 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=2 budget=40960 expected_recv=64569 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=2 budget=40960 expected_recv=63722 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=40960 expected_recv=65191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=2 budget=40960 expected_recv=65191 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=40960 expected_recv=64639 expected_overflow=0 exact_recv_per_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=40960 expected_recv=64681 expected_overflow=0 exact_recv_per_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=2 budget=40960 expected_recv=64639 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=40960 expected_recv=65129 expected_overflow=0 exact_recv_per_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=2 budget=40960 expected_recv=64681 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=2 budget=40960 expected_recv=65129 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=40960 expected_recv=63839 expected_overflow=0 exact_recv_per_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=2 budget=40960 expected_recv=63839 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=40960 expected_recv=65914 expected_overflow=0 exact_recv_per_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=2 budget=40960 expected_recv=65914 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=40960 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=2 budget=40960 expected_recv=65539 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65718 expected_overflow=0 exact_recv_per_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=2 budget=40960 expected_recv=65718 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=40960 expected_recv=64037 expected_overflow=0 exact_recv_per_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=2 budget=40960 expected_recv=64037 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=40960 expected_recv=65508 expected_overflow=0 exact_recv_per_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=2 budget=40960 expected_recv=65508 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=40960 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=2 budget=40960 expected_recv=63568 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=40960 expected_recv=65437 expected_overflow=0 exact_recv_per_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=40960 expected_recv=64879 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=2 budget=40960 expected_recv=65437 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=40960 expected_recv=64883 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=2 budget=40960 expected_recv=64879 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=2 budget=40960 expected_recv=64883 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=40960 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=2 budget=40960 expected_recv=66190 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=40960 expected_recv=65163 expected_overflow=0 exact_recv_per_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=2 budget=40960 expected_recv=65163 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=40960 expected_recv=65401 expected_overflow=0 exact_recv_per_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=40960 expected_recv=62763 expected_overflow=0 exact_recv_per_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=2 budget=40960 expected_recv=65401 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=2 budget=40960 expected_recv=62763 actual_valid_recv=63616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=63616 slack_slots=18304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=40960 expected_recv=64192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=2 budget=40960 expected_recv=64192 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=40960 expected_recv=65095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=40960 expected_recv=65569 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=2 budget=40960 expected_recv=65095 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=2 budget=40960 expected_recv=65569 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=40960 expected_recv=64664 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65491 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=2 budget=40960 expected_recv=64664 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65491 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65204 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=40960 expected_recv=64878 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=12 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=12 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65204 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=12 calls=2 budget=40960 expected_recv=64878 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=40960 expected_recv=64557 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=12 calls=2 budget=40960 expected_recv=64557 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=40960 expected_recv=64433 expected_overflow=0 exact_recv_per_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=13 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=2 budget=40960 expected_recv=64433 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=40960 expected_recv=64784 expected_overflow=0 exact_recv_per_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=40960 expected_recv=63750 expected_overflow=0 exact_recv_per_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=13 calls=2 budget=40960 expected_recv=64784 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=2 budget=40960 expected_recv=63750 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=40960 expected_recv=65057 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=14 calls=2 budget=40960 expected_recv=65057 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=40960 expected_recv=65320 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=2 budget=40960 expected_recv=65320 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=14 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=40960 expected_recv=64175 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=40960 expected_recv=64967 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=14 calls=2 budget=40960 expected_recv=64175 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=14 calls=2 budget=40960 expected_recv=64967 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=40960 expected_recv=65075 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=2 budget=40960 expected_recv=65075 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=40960 expected_recv=63569 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=15 calls=2 budget=40960 expected_recv=63569 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=40960 expected_recv=64684 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=2 budget=40960 expected_recv=64684 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 26996, 22461, 22979, 50624, 30904, 43374] avg=32768.0000 max=50624 max_rank=5 min=22461 min_rank=3 imbalance_ratio=1.544922 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 26996, 22461, 22979, 50624, 30904, 43374] avg=32768.0000 max=50624 max_rank=5 min=22461 min_rank=3 imbalance_ratio=1.544922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] avg=32768.0000 max=33097 max_rank=3 min=31750 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=17527 max_load_reduction_pct=0.346219 moved_to_replicas=58431 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 26996, 22461, 22979, 50624, 30904, 43374] avg=32768.0000 max=50624 max_rank=5 min=22461 min_rank=3 imbalance_ratio=1.544922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 26996, 22461, 22979, 50624, 30904, 43374] avg=32768.0000 max=50624 max_rank=5 min=22461 min_rank=3 imbalance_ratio=1.544922 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65583 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 26996, 22461, 22979, 50624, 30904, 43374] avg=32768.0000 max=50624 max_rank=5 min=22461 min_rank=3 imbalance_ratio=1.544922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 26996, 22461, 22979, 50624, 30904, 43374] avg=32768.0000 max=50624 max_rank=5 min=22461 min_rank=3 imbalance_ratio=1.544922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] avg=32768.0000 max=33097 max_rank=3 min=31750 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=17527 max_load_reduction_pct=0.346219 moved_to_replicas=58431 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 26996, 22461, 22979, 50624, 30904, 43374] avg=32768.0000 max=50624 max_rank=5 min=22461 min_rank=3 imbalance_ratio=1.544922 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=12 calls=2 budget=40960 expected_recv=65583 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[16668, 73537, 11299, 39557, 40193, 2514, 45521, 32855] avg=32768.0000 max=73537 max_rank=1 min=2514 min_rank=5 imbalance_ratio=2.244171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] avg=32768.0000 max=33097 max_rank=3 min=31750 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=17527 max_load_reduction_pct=0.346219 moved_to_replicas=58431 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] avg=32768.0000 max=33097 max_rank=3 min=31750 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=17527 max_load_reduction_pct=0.346219 moved_to_replicas=58431 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] avg=32768.0000 max=33097 max_rank=3 min=31750 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=17527 max_load_reduction_pct=0.346219 moved_to_replicas=58431 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] avg=32768.0000 max=33097 max_rank=3 min=31750 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=17527 max_load_reduction_pct=0.346219 moved_to_replicas=58431 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] avg=32768.0000 max=33097 max_rank=3 min=31750 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=17527 max_load_reduction_pct=0.346219 moved_to_replicas=58431 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[16668, 73537, 11299, 39557, 40193, 2514, 45521, 32855] avg=32768.0000 max=73537 max_rank=1 min=2514 min_rank=5 imbalance_ratio=2.244171 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=40960 expected_recv=64795 expected_overflow=0 exact_recv_per_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] avg=32768.0000 max=33097 max_rank=1 min=30712 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40440 max_load_reduction_pct=0.549927 moved_to_replicas=117254 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=13 calls=2 budget=40960 expected_recv=64795 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[16668, 73537, 11299, 39557, 40193, 2514, 45521, 32855] avg=32768.0000 max=73537 max_rank=1 min=2514 min_rank=5 imbalance_ratio=2.244171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] avg=32768.0000 max=33097 max_rank=1 min=30712 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40440 max_load_reduction_pct=0.549927 moved_to_replicas=117254 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[16668, 73537, 11299, 39557, 40193, 2514, 45521, 32855] avg=32768.0000 max=73537 max_rank=1 min=2514 min_rank=5 imbalance_ratio=2.244171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[16668, 73537, 11299, 39557, 40193, 2514, 45521, 32855] avg=32768.0000 max=73537 max_rank=1 min=2514 min_rank=5 imbalance_ratio=2.244171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[16668, 73537, 11299, 39557, 40193, 2514, 45521, 32855] avg=32768.0000 max=73537 max_rank=1 min=2514 min_rank=5 imbalance_ratio=2.244171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[16668, 73537, 11299, 39557, 40193, 2514, 45521, 32855] avg=32768.0000 max=73537 max_rank=1 min=2514 min_rank=5 imbalance_ratio=2.244171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[58778, 32211, 41142, 29026, 53735, 18463, 7817, 20972] avg=32768.0000 max=58778 max_rank=0 min=7817 min_rank=6 imbalance_ratio=1.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] avg=32768.0000 max=33097 max_rank=1 min=30712 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40440 max_load_reduction_pct=0.549927 moved_to_replicas=117254 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] avg=32768.0000 max=33097 max_rank=1 min=30712 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40440 max_load_reduction_pct=0.549927 moved_to_replicas=117254 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] avg=32768.0000 max=33097 max_rank=1 min=30712 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40440 max_load_reduction_pct=0.549927 moved_to_replicas=117254 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] avg=32768.0000 max=33097 max_rank=1 min=30712 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40440 max_load_reduction_pct=0.549927 moved_to_replicas=117254 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[58778, 32211, 41142, 29026, 53735, 18463, 7817, 20972] avg=32768.0000 max=58778 max_rank=0 min=7817 min_rank=6 imbalance_ratio=1.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] avg=32768.0000 max=33097 max_rank=1 min=30712 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40440 max_load_reduction_pct=0.549927 moved_to_replicas=117254 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31585 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25681 max_load_reduction_pct=0.436915 moved_to_replicas=96118 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31585 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25681 max_load_reduction_pct=0.436915 moved_to_replicas=96118 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[58778, 32211, 41142, 29026, 53735, 18463, 7817, 20972] avg=32768.0000 max=58778 max_rank=0 min=7817 min_rank=6 imbalance_ratio=1.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[58778, 32211, 41142, 29026, 53735, 18463, 7817, 20972] avg=32768.0000 max=58778 max_rank=0 min=7817 min_rank=6 imbalance_ratio=1.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[58778, 32211, 41142, 29026, 53735, 18463, 7817, 20972] avg=32768.0000 max=58778 max_rank=0 min=7817 min_rank=6 imbalance_ratio=1.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[58778, 32211, 41142, 29026, 53735, 18463, 7817, 20972] avg=32768.0000 max=58778 max_rank=0 min=7817 min_rank=6 imbalance_ratio=1.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[39478, 24583, 37640, 47248, 26898, 21200, 41111, 23986] avg=32768.0000 max=47248 max_rank=3 min=21200 min_rank=5 imbalance_ratio=1.441895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[58778, 32211, 41142, 29026, 53735, 18463, 7817, 20972] avg=32768.0000 max=58778 max_rank=0 min=7817 min_rank=6 imbalance_ratio=1.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31585 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25681 max_load_reduction_pct=0.436915 moved_to_replicas=96118 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31585 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25681 max_load_reduction_pct=0.436915 moved_to_replicas=96118 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31585 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25681 max_load_reduction_pct=0.436915 moved_to_replicas=96118 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31585 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25681 max_load_reduction_pct=0.436915 moved_to_replicas=96118 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[39478, 24583, 37640, 47248, 26898, 21200, 41111, 23986] avg=32768.0000 max=47248 max_rank=3 min=21200 min_rank=5 imbalance_ratio=1.441895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] avg=32768.0000 max=33096 max_rank=0 min=31442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14152 max_load_reduction_pct=0.299526 moved_to_replicas=73219 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31585 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25681 max_load_reduction_pct=0.436915 moved_to_replicas=96118 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] avg=32768.0000 max=33096 max_rank=0 min=31442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14152 max_load_reduction_pct=0.299526 moved_to_replicas=73219 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[39478, 24583, 37640, 47248, 26898, 21200, 41111, 23986] avg=32768.0000 max=47248 max_rank=3 min=21200 min_rank=5 imbalance_ratio=1.441895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[39478, 24583, 37640, 47248, 26898, 21200, 41111, 23986] avg=32768.0000 max=47248 max_rank=3 min=21200 min_rank=5 imbalance_ratio=1.441895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[39478, 24583, 37640, 47248, 26898, 21200, 41111, 23986] avg=32768.0000 max=47248 max_rank=3 min=21200 min_rank=5 imbalance_ratio=1.441895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[39478, 24583, 37640, 47248, 26898, 21200, 41111, 23986] avg=32768.0000 max=47248 max_rank=3 min=21200 min_rank=5 imbalance_ratio=1.441895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[43778, 33972, 41269, 27420, 22573, 31464, 23303, 38365] avg=32768.0000 max=43778 max_rank=0 min=22573 min_rank=4 imbalance_ratio=1.335999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[39478, 24583, 37640, 47248, 26898, 21200, 41111, 23986] avg=32768.0000 max=47248 max_rank=3 min=21200 min_rank=5 imbalance_ratio=1.441895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 26996, 22461, 22979, 50624, 30904, 43374] avg=32768.0000 max=50624 max_rank=5 min=22461 min_rank=3 imbalance_ratio=1.544922 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] avg=32768.0000 max=33096 max_rank=0 min=31442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14152 max_load_reduction_pct=0.299526 moved_to_replicas=73219 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] avg=32768.0000 max=33096 max_rank=0 min=31442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14152 max_load_reduction_pct=0.299526 moved_to_replicas=73219 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] avg=32768.0000 max=33096 max_rank=0 min=31442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14152 max_load_reduction_pct=0.299526 moved_to_replicas=73219 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] avg=32768.0000 max=33096 max_rank=0 min=31442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14152 max_load_reduction_pct=0.299526 moved_to_replicas=73219 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[43778, 33972, 41269, 27420, 22573, 31464, 23303, 38365] avg=32768.0000 max=43778 max_rank=0 min=22573 min_rank=4 imbalance_ratio=1.335999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] avg=32768.0000 max=33098 max_rank=0 min=31634 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=10680 max_load_reduction_pct=0.243958 moved_to_replicas=61433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] avg=32768.0000 max=33096 max_rank=0 min=31442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14152 max_load_reduction_pct=0.299526 moved_to_replicas=73219 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[33056, 31750, 33096, 33097, 33096, 33095, 31858, 33096] avg=32768.0000 max=33097 max_rank=3 min=31750 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=17527 max_load_reduction_pct=0.346219 moved_to_replicas=58431 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] avg=32768.0000 max=33098 max_rank=0 min=31634 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=10680 max_load_reduction_pct=0.243958 moved_to_replicas=61433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[43778, 33972, 41269, 27420, 22573, 31464, 23303, 38365] avg=32768.0000 max=43778 max_rank=0 min=22573 min_rank=4 imbalance_ratio=1.335999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[43778, 33972, 41269, 27420, 22573, 31464, 23303, 38365] avg=32768.0000 max=43778 max_rank=0 min=22573 min_rank=4 imbalance_ratio=1.335999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[43778, 33972, 41269, 27420, 22573, 31464, 23303, 38365] avg=32768.0000 max=43778 max_rank=0 min=22573 min_rank=4 imbalance_ratio=1.335999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[43778, 33972, 41269, 27420, 22573, 31464, 23303, 38365] avg=32768.0000 max=43778 max_rank=0 min=22573 min_rank=4 imbalance_ratio=1.335999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[12266, 35078, 70503, 54837, 12318, 10254, 49173, 17715] avg=32768.0000 max=70503 max_rank=2 min=10254 min_rank=5 imbalance_ratio=2.151581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[43778, 33972, 41269, 27420, 22573, 31464, 23303, 38365] avg=32768.0000 max=43778 max_rank=0 min=22573 min_rank=4 imbalance_ratio=1.335999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[16668, 73537, 11299, 39557, 40193, 2514, 45521, 32855] avg=32768.0000 max=73537 max_rank=1 min=2514 min_rank=5 imbalance_ratio=2.244171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] avg=32768.0000 max=33098 max_rank=0 min=31634 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=10680 max_load_reduction_pct=0.243958 moved_to_replicas=61433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] avg=32768.0000 max=33098 max_rank=0 min=31634 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=10680 max_load_reduction_pct=0.243958 moved_to_replicas=61433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] avg=32768.0000 max=33098 max_rank=0 min=31634 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=10680 max_load_reduction_pct=0.243958 moved_to_replicas=61433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] avg=32768.0000 max=33098 max_rank=0 min=31634 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=10680 max_load_reduction_pct=0.243958 moved_to_replicas=61433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[12266, 35078, 70503, 54837, 12318, 10254, 49173, 17715] avg=32768.0000 max=70503 max_rank=2 min=10254 min_rank=5 imbalance_ratio=2.151581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=30471 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37406 max_load_reduction_pct=0.530559 moved_to_replicas=150937 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] avg=32768.0000 max=33098 max_rank=0 min=31634 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=10680 max_load_reduction_pct=0.243958 moved_to_replicas=61433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[30712, 33097, 33095, 33096, 33095, 33097, 33097, 32855] avg=32768.0000 max=33097 max_rank=1 min=30712 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=40440 max_load_reduction_pct=0.549927 moved_to_replicas=117254 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=30471 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37406 max_load_reduction_pct=0.530559 moved_to_replicas=150937 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[12266, 35078, 70503, 54837, 12318, 10254, 49173, 17715] avg=32768.0000 max=70503 max_rank=2 min=10254 min_rank=5 imbalance_ratio=2.151581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[12266, 35078, 70503, 54837, 12318, 10254, 49173, 17715] avg=32768.0000 max=70503 max_rank=2 min=10254 min_rank=5 imbalance_ratio=2.151581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[12266, 35078, 70503, 54837, 12318, 10254, 49173, 17715] avg=32768.0000 max=70503 max_rank=2 min=10254 min_rank=5 imbalance_ratio=2.151581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[12266, 35078, 70503, 54837, 12318, 10254, 49173, 17715] avg=32768.0000 max=70503 max_rank=2 min=10254 min_rank=5 imbalance_ratio=2.151581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[41103, 69582, 53793, 28787, 22039, 14246, 25077, 7517] avg=32768.0000 max=69582 max_rank=1 min=7517 min_rank=7 imbalance_ratio=2.123474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[12266, 35078, 70503, 54837, 12318, 10254, 49173, 17715] avg=32768.0000 max=70503 max_rank=2 min=10254 min_rank=5 imbalance_ratio=2.151581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[58778, 32211, 41142, 29026, 53735, 18463, 7817, 20972] avg=32768.0000 max=58778 max_rank=0 min=7817 min_rank=6 imbalance_ratio=1.793762 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=30471 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37406 max_load_reduction_pct=0.530559 moved_to_replicas=150937 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=30471 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37406 max_load_reduction_pct=0.530559 moved_to_replicas=150937 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=30471 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37406 max_load_reduction_pct=0.530559 moved_to_replicas=150937 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=30471 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37406 max_load_reduction_pct=0.530559 moved_to_replicas=150937 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] avg=32768.0000 max=33096 max_rank=0 min=31473 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36486 max_load_reduction_pct=0.524360 moved_to_replicas=113366 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[41103, 69582, 53793, 28787, 22039, 14246, 25077, 7517] avg=32768.0000 max=69582 max_rank=1 min=7517 min_rank=7 imbalance_ratio=2.123474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=30471 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37406 max_load_reduction_pct=0.530559 moved_to_replicas=150937 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32211, 33097, 32868, 33096, 31585, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31585 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=25681 max_load_reduction_pct=0.436915 moved_to_replicas=96118 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[41103, 69582, 53793, 28787, 22039, 14246, 25077, 7517] avg=32768.0000 max=69582 max_rank=1 min=7517 min_rank=7 imbalance_ratio=2.123474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] avg=32768.0000 max=33096 max_rank=0 min=31473 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36486 max_load_reduction_pct=0.524360 moved_to_replicas=113366 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[41103, 69582, 53793, 28787, 22039, 14246, 25077, 7517] avg=32768.0000 max=69582 max_rank=1 min=7517 min_rank=7 imbalance_ratio=2.123474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[41103, 69582, 53793, 28787, 22039, 14246, 25077, 7517] avg=32768.0000 max=69582 max_rank=1 min=7517 min_rank=7 imbalance_ratio=2.123474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[41103, 69582, 53793, 28787, 22039, 14246, 25077, 7517] avg=32768.0000 max=69582 max_rank=1 min=7517 min_rank=7 imbalance_ratio=2.123474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[17432, 34096, 41175, 38016, 25559, 49545, 39563, 16758] avg=32768.0000 max=49545 max_rank=5 min=16758 min_rank=7 imbalance_ratio=1.511993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[41103, 69582, 53793, 28787, 22039, 14246, 25077, 7517] avg=32768.0000 max=69582 max_rank=1 min=7517 min_rank=7 imbalance_ratio=2.123474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[39478, 24583, 37640, 47248, 26898, 21200, 41111, 23986] avg=32768.0000 max=47248 max_rank=3 min=21200 min_rank=5 imbalance_ratio=1.441895 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] avg=32768.0000 max=33096 max_rank=0 min=31473 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36486 max_load_reduction_pct=0.524360 moved_to_replicas=113366 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] avg=32768.0000 max=33096 max_rank=0 min=31473 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36486 max_load_reduction_pct=0.524360 moved_to_replicas=113366 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] avg=32768.0000 max=33096 max_rank=0 min=31473 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36486 max_load_reduction_pct=0.524360 moved_to_replicas=113366 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] avg=32768.0000 max=33096 max_rank=0 min=31473 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36486 max_load_reduction_pct=0.524360 moved_to_replicas=113366 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31542 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16449 max_load_reduction_pct=0.332001 moved_to_replicas=82564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] avg=32768.0000 max=33096 max_rank=0 min=31473 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36486 max_load_reduction_pct=0.524360 moved_to_replicas=113366 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[17432, 34096, 41175, 38016, 25559, 49545, 39563, 16758] avg=32768.0000 max=49545 max_rank=5 min=16758 min_rank=7 imbalance_ratio=1.511993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[33096, 32598, 33096, 33096, 31442, 33096, 33096, 32624] avg=32768.0000 max=33096 max_rank=0 min=31442 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14152 max_load_reduction_pct=0.299526 moved_to_replicas=73219 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[17432, 34096, 41175, 38016, 25559, 49545, 39563, 16758] avg=32768.0000 max=49545 max_rank=5 min=16758 min_rank=7 imbalance_ratio=1.511993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[17432, 34096, 41175, 38016, 25559, 49545, 39563, 16758] avg=32768.0000 max=49545 max_rank=5 min=16758 min_rank=7 imbalance_ratio=1.511993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31542 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16449 max_load_reduction_pct=0.332001 moved_to_replicas=82564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[17432, 34096, 41175, 38016, 25559, 49545, 39563, 16758] avg=32768.0000 max=49545 max_rank=5 min=16758 min_rank=7 imbalance_ratio=1.511993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[24024, 37006, 47284, 42347, 21548, 15751, 36397, 37787] avg=32768.0000 max=47284 max_rank=2 min=15751 min_rank=5 imbalance_ratio=1.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[17432, 34096, 41175, 38016, 25559, 49545, 39563, 16758] avg=32768.0000 max=49545 max_rank=5 min=16758 min_rank=7 imbalance_ratio=1.511993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[17432, 34096, 41175, 38016, 25559, 49545, 39563, 16758] avg=32768.0000 max=49545 max_rank=5 min=16758 min_rank=7 imbalance_ratio=1.511993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[43778, 33972, 41269, 27420, 22573, 31464, 23303, 38365] avg=32768.0000 max=43778 max_rank=0 min=22573 min_rank=4 imbalance_ratio=1.335999 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31542 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16449 max_load_reduction_pct=0.332001 moved_to_replicas=82564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31542 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16449 max_load_reduction_pct=0.332001 moved_to_replicas=82564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31542 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16449 max_load_reduction_pct=0.332001 moved_to_replicas=82564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30942 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14187 max_load_reduction_pct=0.300038 moved_to_replicas=56241 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31542 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16449 max_load_reduction_pct=0.332001 moved_to_replicas=82564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[24024, 37006, 47284, 42347, 21548, 15751, 36397, 37787] avg=32768.0000 max=47284 max_rank=2 min=15751 min_rank=5 imbalance_ratio=1.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31542 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16449 max_load_reduction_pct=0.332001 moved_to_replicas=82564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[33098, 33096, 33097, 32689, 33094, 32340, 31634, 33096] avg=32768.0000 max=33098 max_rank=0 min=31634 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=10680 max_load_reduction_pct=0.243958 moved_to_replicas=61433 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[24024, 37006, 47284, 42347, 21548, 15751, 36397, 37787] avg=32768.0000 max=47284 max_rank=2 min=15751 min_rank=5 imbalance_ratio=1.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[24024, 37006, 47284, 42347, 21548, 15751, 36397, 37787] avg=32768.0000 max=47284 max_rank=2 min=15751 min_rank=5 imbalance_ratio=1.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30942 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14187 max_load_reduction_pct=0.300038 moved_to_replicas=56241 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[12561, 22511, 50842, 22324, 21257, 41284, 45703, 45662] avg=32768.0000 max=50842 max_rank=2 min=12561 min_rank=0 imbalance_ratio=1.551575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[24024, 37006, 47284, 42347, 21548, 15751, 36397, 37787] avg=32768.0000 max=47284 max_rank=2 min=15751 min_rank=5 imbalance_ratio=1.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[24024, 37006, 47284, 42347, 21548, 15751, 36397, 37787] avg=32768.0000 max=47284 max_rank=2 min=15751 min_rank=5 imbalance_ratio=1.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[12266, 35078, 70503, 54837, 12318, 10254, 49173, 17715] avg=32768.0000 max=70503 max_rank=2 min=10254 min_rank=5 imbalance_ratio=2.151581 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[24024, 37006, 47284, 42347, 21548, 15751, 36397, 37787] avg=32768.0000 max=47284 max_rank=2 min=15751 min_rank=5 imbalance_ratio=1.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30942 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14187 max_load_reduction_pct=0.300038 moved_to_replicas=56241 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30942 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14187 max_load_reduction_pct=0.300038 moved_to_replicas=56241 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17745 max_load_reduction_pct=0.349022 moved_to_replicas=71494 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30942 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14187 max_load_reduction_pct=0.300038 moved_to_replicas=56241 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[12561, 22511, 50842, 22324, 21257, 41284, 45703, 45662] avg=32768.0000 max=50842 max_rank=2 min=12561 min_rank=0 imbalance_ratio=1.551575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30942 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14187 max_load_reduction_pct=0.300038 moved_to_replicas=56241 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 30471, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=2 min=30471 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=37406 max_load_reduction_pct=0.530559 moved_to_replicas=150937 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30942 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14187 max_load_reduction_pct=0.300038 moved_to_replicas=56241 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[12561, 22511, 50842, 22324, 21257, 41284, 45703, 45662] avg=32768.0000 max=50842 max_rank=2 min=12561 min_rank=0 imbalance_ratio=1.551575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[12561, 22511, 50842, 22324, 21257, 41284, 45703, 45662] avg=32768.0000 max=50842 max_rank=2 min=12561 min_rank=0 imbalance_ratio=1.551575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17745 max_load_reduction_pct=0.349022 moved_to_replicas=71494 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[8708, 3426, 66610, 15130, 39667, 52261, 20522, 55820] avg=32768.0000 max=66610 max_rank=2 min=3426 min_rank=1 imbalance_ratio=2.032776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[12561, 22511, 50842, 22324, 21257, 41284, 45703, 45662] avg=32768.0000 max=50842 max_rank=2 min=12561 min_rank=0 imbalance_ratio=1.551575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[41103, 69582, 53793, 28787, 22039, 14246, 25077, 7517] avg=32768.0000 max=69582 max_rank=1 min=7517 min_rank=7 imbalance_ratio=2.123474 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17745 max_load_reduction_pct=0.349022 moved_to_replicas=71494 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[12561, 22511, 50842, 22324, 21257, 41284, 45703, 45662] avg=32768.0000 max=50842 max_rank=2 min=12561 min_rank=0 imbalance_ratio=1.551575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[12561, 22511, 50842, 22324, 21257, 41284, 45703, 45662] avg=32768.0000 max=50842 max_rank=2 min=12561 min_rank=0 imbalance_ratio=1.551575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17745 max_load_reduction_pct=0.349022 moved_to_replicas=71494 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] avg=32768.0000 max=33097 max_rank=3 min=31263 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33513 max_load_reduction_pct=0.503123 moved_to_replicas=159673 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17745 max_load_reduction_pct=0.349022 moved_to_replicas=71494 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[8708, 3426, 66610, 15130, 39667, 52261, 20522, 55820] avg=32768.0000 max=66610 max_rank=2 min=3426 min_rank=1 imbalance_ratio=2.032776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[33096, 33096, 33096, 31473, 33096, 33096, 33096, 32095] avg=32768.0000 max=33096 max_rank=0 min=31473 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=36486 max_load_reduction_pct=0.524360 moved_to_replicas=113366 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17745 max_load_reduction_pct=0.349022 moved_to_replicas=71494 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17745 max_load_reduction_pct=0.349022 moved_to_replicas=71494 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[8708, 3426, 66610, 15130, 39667, 52261, 20522, 55820] avg=32768.0000 max=66610 max_rank=2 min=3426 min_rank=1 imbalance_ratio=2.032776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[8708, 3426, 66610, 15130, 39667, 52261, 20522, 55820] avg=32768.0000 max=66610 max_rank=2 min=3426 min_rank=1 imbalance_ratio=2.032776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] avg=32768.0000 max=33097 max_rank=3 min=31263 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33513 max_load_reduction_pct=0.503123 moved_to_replicas=159673 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[35982, 45267, 18285, 37998, 69653, 13599, 24684, 16676] avg=32768.0000 max=69653 max_rank=4 min=13599 min_rank=5 imbalance_ratio=2.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[8708, 3426, 66610, 15130, 39667, 52261, 20522, 55820] avg=32768.0000 max=66610 max_rank=2 min=3426 min_rank=1 imbalance_ratio=2.032776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[17432, 34096, 41175, 38016, 25559, 49545, 39563, 16758] avg=32768.0000 max=49545 max_rank=5 min=16758 min_rank=7 imbalance_ratio=1.511993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] avg=32768.0000 max=33097 max_rank=3 min=31263 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33513 max_load_reduction_pct=0.503123 moved_to_replicas=159673 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[8708, 3426, 66610, 15130, 39667, 52261, 20522, 55820] avg=32768.0000 max=66610 max_rank=2 min=3426 min_rank=1 imbalance_ratio=2.032776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[8708, 3426, 66610, 15130, 39667, 52261, 20522, 55820] avg=32768.0000 max=66610 max_rank=2 min=3426 min_rank=1 imbalance_ratio=2.032776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] avg=32768.0000 max=33097 max_rank=3 min=31263 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33513 max_load_reduction_pct=0.503123 moved_to_replicas=159673 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] avg=32768.0000 max=33096 max_rank=0 min=31096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36557 max_load_reduction_pct=0.524845 moved_to_replicas=103537 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] avg=32768.0000 max=33097 max_rank=3 min=31263 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33513 max_load_reduction_pct=0.503123 moved_to_replicas=159673 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[31542, 33096, 33096, 33096, 32026, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31542 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16449 max_load_reduction_pct=0.332001 moved_to_replicas=82564 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[35982, 45267, 18285, 37998, 69653, 13599, 24684, 16676] avg=32768.0000 max=69653 max_rank=4 min=13599 min_rank=5 imbalance_ratio=2.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] avg=32768.0000 max=33097 max_rank=3 min=31263 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33513 max_load_reduction_pct=0.503123 moved_to_replicas=159673 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] avg=32768.0000 max=33097 max_rank=3 min=31263 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33513 max_load_reduction_pct=0.503123 moved_to_replicas=159673 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[35982, 45267, 18285, 37998, 69653, 13599, 24684, 16676] avg=32768.0000 max=69653 max_rank=4 min=13599 min_rank=5 imbalance_ratio=2.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[35982, 45267, 18285, 37998, 69653, 13599, 24684, 16676] avg=32768.0000 max=69653 max_rank=4 min=13599 min_rank=5 imbalance_ratio=2.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] avg=32768.0000 max=33096 max_rank=0 min=31096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36557 max_load_reduction_pct=0.524845 moved_to_replicas=103537 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[35303, 24767, 59355, 24403, 54076, 15485, 19501, 29254] avg=32768.0000 max=59355 max_rank=2 min=15485 min_rank=5 imbalance_ratio=1.811371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[24024, 37006, 47284, 42347, 21548, 15751, 36397, 37787] avg=32768.0000 max=47284 max_rank=2 min=15751 min_rank=5 imbalance_ratio=1.442993 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] avg=32768.0000 max=33096 max_rank=0 min=31096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36557 max_load_reduction_pct=0.524845 moved_to_replicas=103537 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[35982, 45267, 18285, 37998, 69653, 13599, 24684, 16676] avg=32768.0000 max=69653 max_rank=4 min=13599 min_rank=5 imbalance_ratio=2.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[35982, 45267, 18285, 37998, 69653, 13599, 24684, 16676] avg=32768.0000 max=69653 max_rank=4 min=13599 min_rank=5 imbalance_ratio=2.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[35982, 45267, 18285, 37998, 69653, 13599, 24684, 16676] avg=32768.0000 max=69653 max_rank=4 min=13599 min_rank=5 imbalance_ratio=2.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] avg=32768.0000 max=33096 max_rank=0 min=31096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36557 max_load_reduction_pct=0.524845 moved_to_replicas=103537 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] avg=32768.0000 max=33098 max_rank=3 min=31461 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=26257 max_load_reduction_pct=0.442372 moved_to_replicas=128555 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[32624, 33096, 33097, 33097, 30942, 33095, 33096, 33097] avg=32768.0000 max=33097 max_rank=2 min=30942 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=14187 max_load_reduction_pct=0.300038 moved_to_replicas=56241 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] avg=32768.0000 max=33096 max_rank=0 min=31096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36557 max_load_reduction_pct=0.524845 moved_to_replicas=103537 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[35303, 24767, 59355, 24403, 54076, 15485, 19501, 29254] avg=32768.0000 max=59355 max_rank=2 min=15485 min_rank=5 imbalance_ratio=1.811371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] avg=32768.0000 max=33096 max_rank=0 min=31096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36557 max_load_reduction_pct=0.524845 moved_to_replicas=103537 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] avg=32768.0000 max=33096 max_rank=0 min=31096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36557 max_load_reduction_pct=0.524845 moved_to_replicas=103537 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[35303, 24767, 59355, 24403, 54076, 15485, 19501, 29254] avg=32768.0000 max=59355 max_rank=2 min=15485 min_rank=5 imbalance_ratio=1.811371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[35303, 24767, 59355, 24403, 54076, 15485, 19501, 29254] avg=32768.0000 max=59355 max_rank=2 min=15485 min_rank=5 imbalance_ratio=1.811371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] avg=32768.0000 max=33098 max_rank=3 min=31461 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=26257 max_load_reduction_pct=0.442372 moved_to_replicas=128555 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[48617, 15158, 39623, 26023, 30560, 32026, 24898, 45239] avg=32768.0000 max=48617 max_rank=0 min=15158 min_rank=1 imbalance_ratio=1.483673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[12561, 22511, 50842, 22324, 21257, 41284, 45703, 45662] avg=32768.0000 max=50842 max_rank=2 min=12561 min_rank=0 imbalance_ratio=1.551575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] avg=32768.0000 max=33098 max_rank=3 min=31461 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=26257 max_load_reduction_pct=0.442372 moved_to_replicas=128555 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[35303, 24767, 59355, 24403, 54076, 15485, 19501, 29254] avg=32768.0000 max=59355 max_rank=2 min=15485 min_rank=5 imbalance_ratio=1.811371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[35303, 24767, 59355, 24403, 54076, 15485, 19501, 29254] avg=32768.0000 max=59355 max_rank=2 min=15485 min_rank=5 imbalance_ratio=1.811371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] avg=32768.0000 max=33098 max_rank=3 min=31461 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=26257 max_load_reduction_pct=0.442372 moved_to_replicas=128555 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[30472, 33096, 33096, 33097, 33096, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=3 min=30472 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=17745 max_load_reduction_pct=0.349022 moved_to_replicas=71494 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=15521 max_load_reduction_pct=0.319250 moved_to_replicas=60695 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[35303, 24767, 59355, 24403, 54076, 15485, 19501, 29254] avg=32768.0000 max=59355 max_rank=2 min=15485 min_rank=5 imbalance_ratio=1.811371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] avg=32768.0000 max=33098 max_rank=3 min=31461 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=26257 max_load_reduction_pct=0.442372 moved_to_replicas=128555 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[48617, 15158, 39623, 26023, 30560, 32026, 24898, 45239] avg=32768.0000 max=48617 max_rank=0 min=15158 min_rank=1 imbalance_ratio=1.483673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] avg=32768.0000 max=33098 max_rank=3 min=31461 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=26257 max_load_reduction_pct=0.442372 moved_to_replicas=128555 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[48617, 15158, 39623, 26023, 30560, 32026, 24898, 45239] avg=32768.0000 max=48617 max_rank=0 min=15158 min_rank=1 imbalance_ratio=1.483673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[48617, 15158, 39623, 26023, 30560, 32026, 24898, 45239] avg=32768.0000 max=48617 max_rank=0 min=15158 min_rank=1 imbalance_ratio=1.483673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] avg=32768.0000 max=33098 max_rank=3 min=31461 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=26257 max_load_reduction_pct=0.442372 moved_to_replicas=128555 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=15521 max_load_reduction_pct=0.319250 moved_to_replicas=60695 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[8708, 3426, 66610, 15130, 39667, 52261, 20522, 55820] avg=32768.0000 max=66610 max_rank=2 min=3426 min_rank=1 imbalance_ratio=2.032776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=15521 max_load_reduction_pct=0.319250 moved_to_replicas=60695 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[37059, 59975, 34617, 42948, 40846, 25572, 3285, 17842] avg=32768.0000 max=59975 max_rank=1 min=3285 min_rank=6 imbalance_ratio=1.830292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[48617, 15158, 39623, 26023, 30560, 32026, 24898, 45239] avg=32768.0000 max=48617 max_rank=0 min=15158 min_rank=1 imbalance_ratio=1.483673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[48617, 15158, 39623, 26023, 30560, 32026, 24898, 45239] avg=32768.0000 max=48617 max_rank=0 min=15158 min_rank=1 imbalance_ratio=1.483673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=15521 max_load_reduction_pct=0.319250 moved_to_replicas=60695 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[33095, 31263, 33095, 33097, 33096, 33096, 32305, 33097] avg=32768.0000 max=33097 max_rank=3 min=31263 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=33513 max_load_reduction_pct=0.503123 moved_to_replicas=159673 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] avg=32768.0000 max=33097 max_rank=3 min=31696 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26878 max_load_reduction_pct=0.448153 moved_to_replicas=88854 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[48617, 15158, 39623, 26023, 30560, 32026, 24898, 45239] avg=32768.0000 max=48617 max_rank=0 min=15158 min_rank=1 imbalance_ratio=1.483673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=15521 max_load_reduction_pct=0.319250 moved_to_replicas=60695 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[37059, 59975, 34617, 42948, 40846, 25572, 3285, 17842] avg=32768.0000 max=59975 max_rank=1 min=3285 min_rank=6 imbalance_ratio=1.830292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=15521 max_load_reduction_pct=0.319250 moved_to_replicas=60695 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[37059, 59975, 34617, 42948, 40846, 25572, 3285, 17842] avg=32768.0000 max=59975 max_rank=1 min=3285 min_rank=6 imbalance_ratio=1.830292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=15521 max_load_reduction_pct=0.319250 moved_to_replicas=60695 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[37059, 59975, 34617, 42948, 40846, 25572, 3285, 17842] avg=32768.0000 max=59975 max_rank=1 min=3285 min_rank=6 imbalance_ratio=1.830292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] avg=32768.0000 max=33097 max_rank=3 min=31696 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26878 max_load_reduction_pct=0.448153 moved_to_replicas=88854 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[35982, 45267, 18285, 37998, 69653, 13599, 24684, 16676] avg=32768.0000 max=69653 max_rank=4 min=13599 min_rank=5 imbalance_ratio=2.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] avg=32768.0000 max=33097 max_rank=3 min=31696 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26878 max_load_reduction_pct=0.448153 moved_to_replicas=88854 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[49021, 21859, 25132, 20425, 29293, 39739, 42603, 34072] avg=32768.0000 max=49021 max_rank=0 min=20425 min_rank=3 imbalance_ratio=1.496002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[37059, 59975, 34617, 42948, 40846, 25572, 3285, 17842] avg=32768.0000 max=59975 max_rank=1 min=3285 min_rank=6 imbalance_ratio=1.830292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[37059, 59975, 34617, 42948, 40846, 25572, 3285, 17842] avg=32768.0000 max=59975 max_rank=1 min=3285 min_rank=6 imbalance_ratio=1.830292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[33096, 33096, 31096, 33095, 33096, 33096, 32473, 33096] avg=32768.0000 max=33096 max_rank=0 min=31096 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=36557 max_load_reduction_pct=0.524845 moved_to_replicas=103537 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] avg=32768.0000 max=33097 max_rank=3 min=31696 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26878 max_load_reduction_pct=0.448153 moved_to_replicas=88854 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[37059, 59975, 34617, 42948, 40846, 25572, 3285, 17842] avg=32768.0000 max=59975 max_rank=1 min=3285 min_rank=6 imbalance_ratio=1.830292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=15925 max_load_reduction_pct=0.324861 moved_to_replicas=61856 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[49021, 21859, 25132, 20425, 29293, 39739, 42603, 34072] avg=32768.0000 max=49021 max_rank=0 min=20425 min_rank=3 imbalance_ratio=1.496002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] avg=32768.0000 max=33097 max_rank=3 min=31696 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26878 max_load_reduction_pct=0.448153 moved_to_replicas=88854 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] avg=32768.0000 max=33097 max_rank=3 min=31696 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26878 max_load_reduction_pct=0.448153 moved_to_replicas=88854 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[49021, 21859, 25132, 20425, 29293, 39739, 42603, 34072] avg=32768.0000 max=49021 max_rank=0 min=20425 min_rank=3 imbalance_ratio=1.496002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] avg=32768.0000 max=33097 max_rank=3 min=31696 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26878 max_load_reduction_pct=0.448153 moved_to_replicas=88854 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[35303, 24767, 59355, 24403, 54076, 15485, 19501, 29254] avg=32768.0000 max=59355 max_rank=2 min=15485 min_rank=5 imbalance_ratio=1.811371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=15925 max_load_reduction_pct=0.324861 moved_to_replicas=61856 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[49021, 21859, 25132, 20425, 29293, 39739, 42603, 34072] avg=32768.0000 max=49021 max_rank=0 min=20425 min_rank=3 imbalance_ratio=1.496002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=15925 max_load_reduction_pct=0.324861 moved_to_replicas=61856 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[49021, 21859, 25132, 20425, 29293, 39739, 42603, 34072] avg=32768.0000 max=49021 max_rank=0 min=20425 min_rank=3 imbalance_ratio=1.496002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[49021, 21859, 25132, 20425, 29293, 39739, 42603, 34072] avg=32768.0000 max=49021 max_rank=0 min=20425 min_rank=3 imbalance_ratio=1.496002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[33096, 33095, 33095, 33098, 33095, 33097, 32107, 31461] avg=32768.0000 max=33098 max_rank=3 min=31461 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=26257 max_load_reduction_pct=0.442372 moved_to_replicas=128555 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=15925 max_load_reduction_pct=0.324861 moved_to_replicas=61856 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[49021, 21859, 25132, 20425, 29293, 39739, 42603, 34072] avg=32768.0000 max=49021 max_rank=0 min=20425 min_rank=3 imbalance_ratio=1.496002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=15925 max_load_reduction_pct=0.324861 moved_to_replicas=61856 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=15925 max_load_reduction_pct=0.324861 moved_to_replicas=61856 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=15925 max_load_reduction_pct=0.324861 moved_to_replicas=61856 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[48617, 15158, 39623, 26023, 30560, 32026, 24898, 45239] avg=32768.0000 max=48617 max_rank=0 min=15158 min_rank=1 imbalance_ratio=1.483673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[33096, 31542, 33096, 33096, 33096, 32026, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31542 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=15521 max_load_reduction_pct=0.319250 moved_to_replicas=60695 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[37059, 59975, 34617, 42948, 40846, 25572, 3285, 17842] avg=32768.0000 max=59975 max_rank=1 min=3285 min_rank=6 imbalance_ratio=1.830292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[33096, 33096, 33096, 33097, 33095, 33097, 31696, 31871] avg=32768.0000 max=33097 max_rank=3 min=31696 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26878 max_load_reduction_pct=0.448153 moved_to_replicas=88854 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[49021, 21859, 25132, 20425, 29293, 39739, 42603, 34072] avg=32768.0000 max=49021 max_rank=0 min=20425 min_rank=3 imbalance_ratio=1.496002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30472, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=15925 max_load_reduction_pct=0.324861 moved_to_replicas=61856 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=32 total_expected_recv=1048800 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=251904 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=32 total_expected_recv=1052426 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=249216 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=32 total_expected_recv=1041955 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=257792 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=32 total_expected_recv=1046514 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=255104 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=32 total_expected_recv=1045859 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=255360 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=32 total_expected_recv=1052841 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=247424 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=32 total_expected_recv=1048459 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=251776 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=32 total_expected_recv=1051754 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=248832 peak_demand=33097 required_capacity=1.010040 +Number of parameters in transformer block in billions: 23.22 [2026-08-30 05:10:25] iteration 1/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 27904.7 | throughput per GPU (TFLOP/s/GPU): 11.9 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102622E+01 | loss scale: 1.0 | grad norm: 44.468 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33536 | min safe capacity: 1.0234 | + +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 58091.849609375 | max allocated: 76371.875 | reserved: 78094.0 | max reserved: 78094.0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=40960 expected_recv=66054 expected_overflow=0 exact_recv_per_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=40960 expected_recv=64303 expected_overflow=0 exact_recv_per_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=40960 expected_recv=65318 expected_overflow=0 exact_recv_per_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=40960 expected_recv=63846 expected_overflow=0 exact_recv_per_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=2 budget=40960 expected_recv=66054 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=2 budget=40960 expected_recv=64303 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=2 budget=40960 expected_recv=65318 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=2 budget=40960 expected_recv=63846 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=40960 expected_recv=64893 expected_overflow=0 exact_recv_per_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=40960 expected_recv=64708 expected_overflow=0 exact_recv_per_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=40960 expected_recv=65405 expected_overflow=0 exact_recv_per_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=2 budget=40960 expected_recv=64893 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=40960 expected_recv=64513 expected_overflow=0 exact_recv_per_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=2 budget=40960 expected_recv=64708 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=2 budget=40960 expected_recv=65405 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=2 budget=40960 expected_recv=64513 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=40960 expected_recv=65498 expected_overflow=0 exact_recv_per_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=2 budget=40960 expected_recv=65498 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=40960 expected_recv=64554 expected_overflow=0 exact_recv_per_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=40960 expected_recv=63872 expected_overflow=0 exact_recv_per_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=40960 expected_recv=65594 expected_overflow=0 exact_recv_per_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=40960 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=2 budget=40960 expected_recv=64554 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=2 budget=40960 expected_recv=63872 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=2 budget=40960 expected_recv=65594 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=2 budget=40960 expected_recv=66190 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=40960 expected_recv=65650 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=40960 expected_recv=64882 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=40960 expected_recv=65951 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=2 budget=40960 expected_recv=65650 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=40960 expected_recv=63037 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=2 budget=40960 expected_recv=64882 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=2 budget=40960 expected_recv=65951 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=2 budget=40960 expected_recv=63037 actual_valid_recv=63616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=63616 slack_slots=18304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=40960 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=2 budget=40960 expected_recv=63568 actual_valid_recv=64000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=40960 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=64000 slack_slots=17920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=2 budget=40960 expected_recv=63568 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=40960 expected_recv=65119 expected_overflow=0 exact_recv_per_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=2 budget=40960 expected_recv=65119 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=40960 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=40960 expected_recv=64640 expected_overflow=0 exact_recv_per_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=2 budget=40960 expected_recv=63568 actual_valid_recv=64128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=2 budget=40960 expected_recv=64640 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=64128 slack_slots=17792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=40960 expected_recv=66086 expected_overflow=0 exact_recv_per_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=40960 expected_recv=66017 expected_overflow=0 exact_recv_per_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=40960 expected_recv=65317 expected_overflow=0 exact_recv_per_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=40960 expected_recv=62098 expected_overflow=0 exact_recv_per_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=2 budget=40960 expected_recv=66086 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=2 budget=40960 expected_recv=65317 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=2 budget=40960 expected_recv=66017 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=2 budget=40960 expected_recv=62098 actual_valid_recv=62848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=62848 slack_slots=19072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=40960 expected_recv=65504 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=2 budget=40960 expected_recv=65504 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=40960 expected_recv=63566 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=40960 expected_recv=64256 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=2 budget=40960 expected_recv=63566 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=2 budget=40960 expected_recv=64256 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=40960 expected_recv=64605 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=40960 expected_recv=64138 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=2 budget=40960 expected_recv=64605 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=2 budget=40960 expected_recv=64138 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=40960 expected_recv=64585 expected_overflow=0 exact_recv_per_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=2 budget=40960 expected_recv=64585 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=40960 expected_recv=65235 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=40960 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=2 budget=40960 expected_recv=65235 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=40960 expected_recv=61899 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=2 budget=40960 expected_recv=63568 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=2 budget=40960 expected_recv=61899 actual_valid_recv=62976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=62976 slack_slots=18944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=40960 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=2 budget=40960 expected_recv=63568 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=40960 expected_recv=63704 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=40960 expected_recv=64773 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=2 budget=40960 expected_recv=63704 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=40960 expected_recv=64852 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=2 budget=40960 expected_recv=64773 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=40960 expected_recv=64232 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=2 budget=40960 expected_recv=64852 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=12 calls=2 budget=40960 expected_recv=64232 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=12 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=40960 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=40960 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=12 calls=2 budget=40960 expected_recv=63568 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65264 expected_overflow=0 exact_recv_per_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=12 calls=2 budget=40960 expected_recv=65527 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65264 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=13 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65128 expected_overflow=0 exact_recv_per_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65128 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=40960 expected_recv=64743 expected_overflow=0 exact_recv_per_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=13 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=13 calls=2 budget=40960 expected_recv=64743 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65635 expected_overflow=0 exact_recv_per_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=40960 expected_recv=64942 expected_overflow=0 exact_recv_per_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=14 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=2 budget=40960 expected_recv=65635 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=2 budget=40960 expected_recv=64942 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=40960 expected_recv=63766 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=14 calls=2 budget=40960 expected_recv=63766 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=40960 expected_recv=65822 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=14 calls=2 budget=40960 expected_recv=65822 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=40960 expected_recv=66084 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=40960 expected_recv=65185 expected_overflow=0 exact_recv_per_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=2 budget=40960 expected_recv=66084 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=14 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=40960 expected_recv=63849 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=15 calls=2 budget=40960 expected_recv=65185 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=14 calls=2 budget=40960 expected_recv=63849 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=15 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=40960 expected_recv=61952 expected_overflow=0 exact_recv_per_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[32819, 30237, 26421, 20996, 22180, 51615, 30887, 46989] avg=32768.0000 max=51615 max_rank=5 min=20996 min_rank=3 imbalance_ratio=1.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=2 budget=40960 expected_recv=61952 actual_valid_recv=62720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[32819, 30237, 26421, 20996, 22180, 51615, 30887, 46989] avg=32768.0000 max=51615 max_rank=5 min=20996 min_rank=3 imbalance_ratio=1.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=62720 slack_slots=19200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[32819, 30237, 26421, 20996, 22180, 51615, 30887, 46989] avg=32768.0000 max=51615 max_rank=5 min=20996 min_rank=3 imbalance_ratio=1.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] avg=32768.0000 max=33097 max_rank=5 min=30887 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18518 max_load_reduction_pct=0.358772 moved_to_replicas=61334 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[32819, 30237, 26421, 20996, 22180, 51615, 30887, 46989] avg=32768.0000 max=51615 max_rank=5 min=20996 min_rank=3 imbalance_ratio=1.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] avg=32768.0000 max=33097 max_rank=5 min=30887 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18518 max_load_reduction_pct=0.358772 moved_to_replicas=61334 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] avg=32768.0000 max=33097 max_rank=5 min=30887 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18518 max_load_reduction_pct=0.358772 moved_to_replicas=61334 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[32819, 30237, 26421, 20996, 22180, 51615, 30887, 46989] avg=32768.0000 max=51615 max_rank=5 min=20996 min_rank=3 imbalance_ratio=1.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[32819, 30237, 26421, 20996, 22180, 51615, 30887, 46989] avg=32768.0000 max=51615 max_rank=5 min=20996 min_rank=3 imbalance_ratio=1.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[32819, 30237, 26421, 20996, 22180, 51615, 30887, 46989] avg=32768.0000 max=51615 max_rank=5 min=20996 min_rank=3 imbalance_ratio=1.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[32819, 30237, 26421, 20996, 22180, 51615, 30887, 46989] avg=32768.0000 max=51615 max_rank=5 min=20996 min_rank=3 imbalance_ratio=1.575165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[19440, 68443, 18830, 38447, 36834, 2641, 44572, 32937] avg=32768.0000 max=68443 max_rank=1 min=2641 min_rank=5 imbalance_ratio=2.088715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] avg=32768.0000 max=33097 max_rank=5 min=30887 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18518 max_load_reduction_pct=0.358772 moved_to_replicas=61334 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[19440, 68443, 18830, 38447, 36834, 2641, 44572, 32937] avg=32768.0000 max=68443 max_rank=1 min=2641 min_rank=5 imbalance_ratio=2.088715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] avg=32768.0000 max=33097 max_rank=5 min=30887 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18518 max_load_reduction_pct=0.358772 moved_to_replicas=61334 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] avg=32768.0000 max=33097 max_rank=5 min=30887 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18518 max_load_reduction_pct=0.358772 moved_to_replicas=61334 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] avg=32768.0000 max=33097 max_rank=5 min=30887 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18518 max_load_reduction_pct=0.358772 moved_to_replicas=61334 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[19440, 68443, 18830, 38447, 36834, 2641, 44572, 32937] avg=32768.0000 max=68443 max_rank=1 min=2641 min_rank=5 imbalance_ratio=2.088715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[32819, 32958, 33096, 33095, 33096, 33097, 30887, 33096] avg=32768.0000 max=33097 max_rank=5 min=30887 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=18518 max_load_reduction_pct=0.358772 moved_to_replicas=61334 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] avg=32768.0000 max=33097 max_rank=2 min=31418 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35346 max_load_reduction_pct=0.516430 moved_to_replicas=118233 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] avg=32768.0000 max=33097 max_rank=2 min=31418 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35346 max_load_reduction_pct=0.516430 moved_to_replicas=118233 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[19440, 68443, 18830, 38447, 36834, 2641, 44572, 32937] avg=32768.0000 max=68443 max_rank=1 min=2641 min_rank=5 imbalance_ratio=2.088715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] avg=32768.0000 max=33097 max_rank=2 min=31418 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35346 max_load_reduction_pct=0.516430 moved_to_replicas=118233 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[19440, 68443, 18830, 38447, 36834, 2641, 44572, 32937] avg=32768.0000 max=68443 max_rank=1 min=2641 min_rank=5 imbalance_ratio=2.088715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[19440, 68443, 18830, 38447, 36834, 2641, 44572, 32937] avg=32768.0000 max=68443 max_rank=1 min=2641 min_rank=5 imbalance_ratio=2.088715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[19440, 68443, 18830, 38447, 36834, 2641, 44572, 32937] avg=32768.0000 max=68443 max_rank=1 min=2641 min_rank=5 imbalance_ratio=2.088715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] avg=32768.0000 max=33097 max_rank=2 min=31418 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35346 max_load_reduction_pct=0.516430 moved_to_replicas=118233 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[19440, 68443, 18830, 38447, 36834, 2641, 44572, 32937] avg=32768.0000 max=68443 max_rank=1 min=2641 min_rank=5 imbalance_ratio=2.088715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[67697, 32402, 38323, 24276, 54722, 15725, 9034, 19965] avg=32768.0000 max=67697 max_rank=0 min=9034 min_rank=6 imbalance_ratio=2.065948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[67697, 32402, 38323, 24276, 54722, 15725, 9034, 19965] avg=32768.0000 max=67697 max_rank=0 min=9034 min_rank=6 imbalance_ratio=2.065948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] avg=32768.0000 max=33097 max_rank=2 min=31418 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35346 max_load_reduction_pct=0.516430 moved_to_replicas=118233 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] avg=32768.0000 max=33097 max_rank=2 min=31418 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35346 max_load_reduction_pct=0.516430 moved_to_replicas=118233 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] avg=32768.0000 max=33097 max_rank=2 min=31418 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35346 max_load_reduction_pct=0.516430 moved_to_replicas=118233 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[67697, 32402, 38323, 24276, 54722, 15725, 9034, 19965] avg=32768.0000 max=67697 max_rank=0 min=9034 min_rank=6 imbalance_ratio=2.065948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[31418, 33095, 33097, 33096, 33096, 32309, 33096, 32937] avg=32768.0000 max=33097 max_rank=2 min=31418 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=35346 max_load_reduction_pct=0.516430 moved_to_replicas=118233 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=31490 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34600 max_load_reduction_pct=0.511101 moved_to_replicas=95117 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=31490 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34600 max_load_reduction_pct=0.511101 moved_to_replicas=95117 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[67697, 32402, 38323, 24276, 54722, 15725, 9034, 19965] avg=32768.0000 max=67697 max_rank=0 min=9034 min_rank=6 imbalance_ratio=2.065948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=31490 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34600 max_load_reduction_pct=0.511101 moved_to_replicas=95117 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[67697, 32402, 38323, 24276, 54722, 15725, 9034, 19965] avg=32768.0000 max=67697 max_rank=0 min=9034 min_rank=6 imbalance_ratio=2.065948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[67697, 32402, 38323, 24276, 54722, 15725, 9034, 19965] avg=32768.0000 max=67697 max_rank=0 min=9034 min_rank=6 imbalance_ratio=2.065948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=31490 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34600 max_load_reduction_pct=0.511101 moved_to_replicas=95117 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[67697, 32402, 38323, 24276, 54722, 15725, 9034, 19965] avg=32768.0000 max=67697 max_rank=0 min=9034 min_rank=6 imbalance_ratio=2.065948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[43762, 23555, 26659, 42638, 36575, 21358, 35811, 31786] avg=32768.0000 max=43762 max_rank=0 min=21358 min_rank=5 imbalance_ratio=1.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[67697, 32402, 38323, 24276, 54722, 15725, 9034, 19965] avg=32768.0000 max=67697 max_rank=0 min=9034 min_rank=6 imbalance_ratio=2.065948 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[43762, 23555, 26659, 42638, 36575, 21358, 35811, 31786] avg=32768.0000 max=43762 max_rank=0 min=21358 min_rank=5 imbalance_ratio=1.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=31490 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34600 max_load_reduction_pct=0.511101 moved_to_replicas=95117 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=31490 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34600 max_load_reduction_pct=0.511101 moved_to_replicas=95117 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=31490 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34600 max_load_reduction_pct=0.511101 moved_to_replicas=95117 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[43762, 23555, 26659, 42638, 36575, 21358, 35811, 31786] avg=32768.0000 max=43762 max_rank=0 min=21358 min_rank=5 imbalance_ratio=1.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[43762, 23555, 26659, 42638, 36575, 21358, 35811, 31786] avg=32768.0000 max=43762 max_rank=0 min=21358 min_rank=5 imbalance_ratio=1.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] avg=32768.0000 max=33097 max_rank=0 min=31786 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=10665 max_load_reduction_pct=0.243705 moved_to_replicas=46681 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[33097, 32402, 33096, 32771, 33095, 31490, 33096, 33097] avg=32768.0000 max=33097 max_rank=0 min=31490 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=34600 max_load_reduction_pct=0.511101 moved_to_replicas=95117 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] avg=32768.0000 max=33097 max_rank=0 min=31786 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=10665 max_load_reduction_pct=0.243705 moved_to_replicas=46681 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] avg=32768.0000 max=33097 max_rank=0 min=31786 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=10665 max_load_reduction_pct=0.243705 moved_to_replicas=46681 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] avg=32768.0000 max=33097 max_rank=0 min=31786 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=10665 max_load_reduction_pct=0.243705 moved_to_replicas=46681 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[43762, 23555, 26659, 42638, 36575, 21358, 35811, 31786] avg=32768.0000 max=43762 max_rank=0 min=21358 min_rank=5 imbalance_ratio=1.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[43762, 23555, 26659, 42638, 36575, 21358, 35811, 31786] avg=32768.0000 max=43762 max_rank=0 min=21358 min_rank=5 imbalance_ratio=1.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[43762, 23555, 26659, 42638, 36575, 21358, 35811, 31786] avg=32768.0000 max=43762 max_rank=0 min=21358 min_rank=5 imbalance_ratio=1.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[51993, 40899, 55140, 20227, 11450, 20345, 14148, 47942] avg=32768.0000 max=55140 max_rank=2 min=11450 min_rank=4 imbalance_ratio=1.682739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[43762, 23555, 26659, 42638, 36575, 21358, 35811, 31786] avg=32768.0000 max=43762 max_rank=0 min=21358 min_rank=5 imbalance_ratio=1.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[51993, 40899, 55140, 20227, 11450, 20345, 14148, 47942] avg=32768.0000 max=55140 max_rank=2 min=11450 min_rank=4 imbalance_ratio=1.682739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] avg=32768.0000 max=33097 max_rank=0 min=31786 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=10665 max_load_reduction_pct=0.243705 moved_to_replicas=46681 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] avg=32768.0000 max=33097 max_rank=0 min=31786 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=10665 max_load_reduction_pct=0.243705 moved_to_replicas=46681 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] avg=32768.0000 max=33097 max_rank=0 min=31786 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=10665 max_load_reduction_pct=0.243705 moved_to_replicas=46681 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[51993, 40899, 55140, 20227, 11450, 20345, 14148, 47942] avg=32768.0000 max=55140 max_rank=2 min=11450 min_rank=4 imbalance_ratio=1.682739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[51993, 40899, 55140, 20227, 11450, 20345, 14148, 47942] avg=32768.0000 max=55140 max_rank=2 min=11450 min_rank=4 imbalance_ratio=1.682739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=22043 max_load_reduction_pct=0.399764 moved_to_replicas=112959 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[33097, 33096, 32855, 33096, 33096, 32023, 33095, 31786] avg=32768.0000 max=33097 max_rank=0 min=31786 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=10665 max_load_reduction_pct=0.243705 moved_to_replicas=46681 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=22043 max_load_reduction_pct=0.399764 moved_to_replicas=112959 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=22043 max_load_reduction_pct=0.399764 moved_to_replicas=112959 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=22043 max_load_reduction_pct=0.399764 moved_to_replicas=112959 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[51993, 40899, 55140, 20227, 11450, 20345, 14148, 47942] avg=32768.0000 max=55140 max_rank=2 min=11450 min_rank=4 imbalance_ratio=1.682739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[51993, 40899, 55140, 20227, 11450, 20345, 14148, 47942] avg=32768.0000 max=55140 max_rank=2 min=11450 min_rank=4 imbalance_ratio=1.682739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[51993, 40899, 55140, 20227, 11450, 20345, 14148, 47942] avg=32768.0000 max=55140 max_rank=2 min=11450 min_rank=4 imbalance_ratio=1.682739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[18184, 32023, 50256, 65266, 13240, 7834, 55043, 20298] avg=32768.0000 max=65266 max_rank=3 min=7834 min_rank=5 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[51993, 40899, 55140, 20227, 11450, 20345, 14148, 47942] avg=32768.0000 max=55140 max_rank=2 min=11450 min_rank=4 imbalance_ratio=1.682739 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[18184, 32023, 50256, 65266, 13240, 7834, 55043, 20298] avg=32768.0000 max=65266 max_rank=3 min=7834 min_rank=5 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=22043 max_load_reduction_pct=0.399764 moved_to_replicas=112959 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=22043 max_load_reduction_pct=0.399764 moved_to_replicas=112959 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[18184, 32023, 50256, 65266, 13240, 7834, 55043, 20298] avg=32768.0000 max=65266 max_rank=3 min=7834 min_rank=5 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=22043 max_load_reduction_pct=0.399764 moved_to_replicas=112959 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31544 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32169 max_load_reduction_pct=0.492891 moved_to_replicas=134809 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[18184, 32023, 50256, 65266, 13240, 7834, 55043, 20298] avg=32768.0000 max=65266 max_rank=3 min=7834 min_rank=5 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[33095, 33096, 33096, 30472, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=22043 max_load_reduction_pct=0.399764 moved_to_replicas=112959 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31544 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32169 max_load_reduction_pct=0.492891 moved_to_replicas=134809 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31544 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32169 max_load_reduction_pct=0.492891 moved_to_replicas=134809 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31544 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32169 max_load_reduction_pct=0.492891 moved_to_replicas=134809 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[18184, 32023, 50256, 65266, 13240, 7834, 55043, 20298] avg=32768.0000 max=65266 max_rank=3 min=7834 min_rank=5 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[18184, 32023, 50256, 65266, 13240, 7834, 55043, 20298] avg=32768.0000 max=65266 max_rank=3 min=7834 min_rank=5 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[18184, 32023, 50256, 65266, 13240, 7834, 55043, 20298] avg=32768.0000 max=65266 max_rank=3 min=7834 min_rank=5 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[30358, 29747, 71433, 19440, 48778, 10974, 33360, 18054] avg=32768.0000 max=71433 max_rank=2 min=10974 min_rank=5 imbalance_ratio=2.179962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[18184, 32023, 50256, 65266, 13240, 7834, 55043, 20298] avg=32768.0000 max=65266 max_rank=3 min=7834 min_rank=5 imbalance_ratio=1.991760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[30358, 29747, 71433, 19440, 48778, 10974, 33360, 18054] avg=32768.0000 max=71433 max_rank=2 min=10974 min_rank=5 imbalance_ratio=2.179962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31544 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32169 max_load_reduction_pct=0.492891 moved_to_replicas=134809 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[30358, 29747, 71433, 19440, 48778, 10974, 33360, 18054] avg=32768.0000 max=71433 max_rank=2 min=10974 min_rank=5 imbalance_ratio=2.179962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31544 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32169 max_load_reduction_pct=0.492891 moved_to_replicas=134809 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38336 max_load_reduction_pct=0.536671 moved_to_replicas=91766 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31544 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32169 max_load_reduction_pct=0.492891 moved_to_replicas=134809 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[30358, 29747, 71433, 19440, 48778, 10974, 33360, 18054] avg=32768.0000 max=71433 max_rank=2 min=10974 min_rank=5 imbalance_ratio=2.179962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[31544, 32023, 33097, 33095, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31544 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32169 max_load_reduction_pct=0.492891 moved_to_replicas=134809 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38336 max_load_reduction_pct=0.536671 moved_to_replicas=91766 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38336 max_load_reduction_pct=0.536671 moved_to_replicas=91766 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[30358, 29747, 71433, 19440, 48778, 10974, 33360, 18054] avg=32768.0000 max=71433 max_rank=2 min=10974 min_rank=5 imbalance_ratio=2.179962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38336 max_load_reduction_pct=0.536671 moved_to_replicas=91766 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[30358, 29747, 71433, 19440, 48778, 10974, 33360, 18054] avg=32768.0000 max=71433 max_rank=2 min=10974 min_rank=5 imbalance_ratio=2.179962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[18203, 46751, 40589, 40296, 10305, 43271, 40091, 22638] avg=32768.0000 max=46751 max_rank=1 min=10305 min_rank=4 imbalance_ratio=1.426727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[30358, 29747, 71433, 19440, 48778, 10974, 33360, 18054] avg=32768.0000 max=71433 max_rank=2 min=10974 min_rank=5 imbalance_ratio=2.179962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[30358, 29747, 71433, 19440, 48778, 10974, 33360, 18054] avg=32768.0000 max=71433 max_rank=2 min=10974 min_rank=5 imbalance_ratio=2.179962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[18203, 46751, 40589, 40296, 10305, 43271, 40091, 22638] avg=32768.0000 max=46751 max_rank=1 min=10305 min_rank=4 imbalance_ratio=1.426727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38336 max_load_reduction_pct=0.536671 moved_to_replicas=91766 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[18203, 46751, 40589, 40296, 10305, 43271, 40091, 22638] avg=32768.0000 max=46751 max_rank=1 min=10305 min_rank=4 imbalance_ratio=1.426727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38336 max_load_reduction_pct=0.536671 moved_to_replicas=91766 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] avg=32768.0000 max=33097 max_rank=0 min=31160 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=13654 max_load_reduction_pct=0.292058 moved_to_replicas=106029 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38336 max_load_reduction_pct=0.536671 moved_to_replicas=91766 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[31026, 33097, 33096, 33095, 33096, 32543, 33095, 33096] avg=32768.0000 max=33097 max_rank=1 min=31026 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=38336 max_load_reduction_pct=0.536671 moved_to_replicas=91766 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[18203, 46751, 40589, 40296, 10305, 43271, 40091, 22638] avg=32768.0000 max=46751 max_rank=1 min=10305 min_rank=4 imbalance_ratio=1.426727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] avg=32768.0000 max=33097 max_rank=0 min=31160 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=13654 max_load_reduction_pct=0.292058 moved_to_replicas=106029 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] avg=32768.0000 max=33097 max_rank=0 min=31160 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=13654 max_load_reduction_pct=0.292058 moved_to_replicas=106029 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[18203, 46751, 40589, 40296, 10305, 43271, 40091, 22638] avg=32768.0000 max=46751 max_rank=1 min=10305 min_rank=4 imbalance_ratio=1.426727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] avg=32768.0000 max=33097 max_rank=0 min=31160 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=13654 max_load_reduction_pct=0.292058 moved_to_replicas=106029 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[17522, 36901, 52335, 45105, 20517, 20282, 36223, 33259] avg=32768.0000 max=52335 max_rank=2 min=17522 min_rank=0 imbalance_ratio=1.597137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[18203, 46751, 40589, 40296, 10305, 43271, 40091, 22638] avg=32768.0000 max=46751 max_rank=1 min=10305 min_rank=4 imbalance_ratio=1.426727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[18203, 46751, 40589, 40296, 10305, 43271, 40091, 22638] avg=32768.0000 max=46751 max_rank=1 min=10305 min_rank=4 imbalance_ratio=1.426727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[18203, 46751, 40589, 40296, 10305, 43271, 40091, 22638] avg=32768.0000 max=46751 max_rank=1 min=10305 min_rank=4 imbalance_ratio=1.426727 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[17522, 36901, 52335, 45105, 20517, 20282, 36223, 33259] avg=32768.0000 max=52335 max_rank=2 min=17522 min_rank=0 imbalance_ratio=1.597137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] avg=32768.0000 max=33097 max_rank=0 min=31160 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=13654 max_load_reduction_pct=0.292058 moved_to_replicas=106029 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31043 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=19238 max_load_reduction_pct=0.367593 moved_to_replicas=65890 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[17522, 36901, 52335, 45105, 20517, 20282, 36223, 33259] avg=32768.0000 max=52335 max_rank=2 min=17522 min_rank=0 imbalance_ratio=1.597137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] avg=32768.0000 max=33097 max_rank=0 min=31160 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=13654 max_load_reduction_pct=0.292058 moved_to_replicas=106029 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] avg=32768.0000 max=33097 max_rank=0 min=31160 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=13654 max_load_reduction_pct=0.292058 moved_to_replicas=106029 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[17522, 36901, 52335, 45105, 20517, 20282, 36223, 33259] avg=32768.0000 max=52335 max_rank=2 min=17522 min_rank=0 imbalance_ratio=1.597137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[33097, 33096, 33095, 33096, 31160, 33096, 33095, 32409] avg=32768.0000 max=33097 max_rank=0 min=31160 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=13654 max_load_reduction_pct=0.292058 moved_to_replicas=106029 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31043 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=19238 max_load_reduction_pct=0.367593 moved_to_replicas=65890 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31043 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=19238 max_load_reduction_pct=0.367593 moved_to_replicas=65890 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[17522, 36901, 52335, 45105, 20517, 20282, 36223, 33259] avg=32768.0000 max=52335 max_rank=2 min=17522 min_rank=0 imbalance_ratio=1.597137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31043 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=19238 max_load_reduction_pct=0.367593 moved_to_replicas=65890 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[17141, 38895, 36571, 33772, 24822, 38998, 26341, 45604] avg=32768.0000 max=45604 max_rank=7 min=17141 min_rank=0 imbalance_ratio=1.391724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[17522, 36901, 52335, 45105, 20517, 20282, 36223, 33259] avg=32768.0000 max=52335 max_rank=2 min=17522 min_rank=0 imbalance_ratio=1.597137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[17522, 36901, 52335, 45105, 20517, 20282, 36223, 33259] avg=32768.0000 max=52335 max_rank=2 min=17522 min_rank=0 imbalance_ratio=1.597137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[17522, 36901, 52335, 45105, 20517, 20282, 36223, 33259] avg=32768.0000 max=52335 max_rank=2 min=17522 min_rank=0 imbalance_ratio=1.597137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[17141, 38895, 36571, 33772, 24822, 38998, 26341, 45604] avg=32768.0000 max=45604 max_rank=7 min=17141 min_rank=0 imbalance_ratio=1.391724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31043 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=19238 max_load_reduction_pct=0.367593 moved_to_replicas=65890 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] avg=32768.0000 max=33097 max_rank=1 min=31428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12507 max_load_reduction_pct=0.274252 moved_to_replicas=64817 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[17141, 38895, 36571, 33772, 24822, 38998, 26341, 45604] avg=32768.0000 max=45604 max_rank=7 min=17141 min_rank=0 imbalance_ratio=1.391724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31043 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=19238 max_load_reduction_pct=0.367593 moved_to_replicas=65890 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31043 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=19238 max_load_reduction_pct=0.367593 moved_to_replicas=65890 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[17141, 38895, 36571, 33772, 24822, 38998, 26341, 45604] avg=32768.0000 max=45604 max_rank=7 min=17141 min_rank=0 imbalance_ratio=1.391724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[33097, 33095, 33095, 33096, 32526, 31043, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31043 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=19238 max_load_reduction_pct=0.367593 moved_to_replicas=65890 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] avg=32768.0000 max=33097 max_rank=1 min=31428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12507 max_load_reduction_pct=0.274252 moved_to_replicas=64817 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] avg=32768.0000 max=33097 max_rank=1 min=31428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12507 max_load_reduction_pct=0.274252 moved_to_replicas=64817 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] avg=32768.0000 max=33097 max_rank=1 min=31428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12507 max_load_reduction_pct=0.274252 moved_to_replicas=64817 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[15517, 6296, 56463, 37805, 58988, 49310, 3281, 34484] avg=32768.0000 max=58988 max_rank=4 min=3281 min_rank=6 imbalance_ratio=1.800171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[17141, 38895, 36571, 33772, 24822, 38998, 26341, 45604] avg=32768.0000 max=45604 max_rank=7 min=17141 min_rank=0 imbalance_ratio=1.391724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[17141, 38895, 36571, 33772, 24822, 38998, 26341, 45604] avg=32768.0000 max=45604 max_rank=7 min=17141 min_rank=0 imbalance_ratio=1.391724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[17141, 38895, 36571, 33772, 24822, 38998, 26341, 45604] avg=32768.0000 max=45604 max_rank=7 min=17141 min_rank=0 imbalance_ratio=1.391724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[17141, 38895, 36571, 33772, 24822, 38998, 26341, 45604] avg=32768.0000 max=45604 max_rank=7 min=17141 min_rank=0 imbalance_ratio=1.391724 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[15517, 6296, 56463, 37805, 58988, 49310, 3281, 34484] avg=32768.0000 max=58988 max_rank=4 min=3281 min_rank=6 imbalance_ratio=1.800171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] avg=32768.0000 max=33097 max_rank=7 min=30471 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25891 max_load_reduction_pct=0.438920 moved_to_replicas=147815 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] avg=32768.0000 max=33097 max_rank=1 min=31428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12507 max_load_reduction_pct=0.274252 moved_to_replicas=64817 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[15517, 6296, 56463, 37805, 58988, 49310, 3281, 34484] avg=32768.0000 max=58988 max_rank=4 min=3281 min_rank=6 imbalance_ratio=1.800171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[15517, 6296, 56463, 37805, 58988, 49310, 3281, 34484] avg=32768.0000 max=58988 max_rank=4 min=3281 min_rank=6 imbalance_ratio=1.800171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] avg=32768.0000 max=33097 max_rank=1 min=31428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12507 max_load_reduction_pct=0.274252 moved_to_replicas=64817 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] avg=32768.0000 max=33097 max_rank=1 min=31428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12507 max_load_reduction_pct=0.274252 moved_to_replicas=64817 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[33095, 33097, 33096, 33096, 31428, 33096, 32139, 33097] avg=32768.0000 max=33097 max_rank=1 min=31428 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12507 max_load_reduction_pct=0.274252 moved_to_replicas=64817 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] avg=32768.0000 max=33097 max_rank=7 min=30471 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25891 max_load_reduction_pct=0.438920 moved_to_replicas=147815 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] avg=32768.0000 max=33097 max_rank=7 min=30471 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25891 max_load_reduction_pct=0.438920 moved_to_replicas=147815 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[50913, 44241, 30608, 29223, 47040, 11147, 22660, 26312] avg=32768.0000 max=50913 max_rank=0 min=11147 min_rank=5 imbalance_ratio=1.553741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] avg=32768.0000 max=33097 max_rank=7 min=30471 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25891 max_load_reduction_pct=0.438920 moved_to_replicas=147815 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[15517, 6296, 56463, 37805, 58988, 49310, 3281, 34484] avg=32768.0000 max=58988 max_rank=4 min=3281 min_rank=6 imbalance_ratio=1.800171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[15517, 6296, 56463, 37805, 58988, 49310, 3281, 34484] avg=32768.0000 max=58988 max_rank=4 min=3281 min_rank=6 imbalance_ratio=1.800171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[15517, 6296, 56463, 37805, 58988, 49310, 3281, 34484] avg=32768.0000 max=58988 max_rank=4 min=3281 min_rank=6 imbalance_ratio=1.800171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[50913, 44241, 30608, 29223, 47040, 11147, 22660, 26312] avg=32768.0000 max=50913 max_rank=0 min=11147 min_rank=5 imbalance_ratio=1.553741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[15517, 6296, 56463, 37805, 58988, 49310, 3281, 34484] avg=32768.0000 max=58988 max_rank=4 min=3281 min_rank=6 imbalance_ratio=1.800171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] avg=32768.0000 max=33096 max_rank=0 min=30608 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17817 max_load_reduction_pct=0.349950 moved_to_replicas=99788 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] avg=32768.0000 max=33097 max_rank=7 min=30471 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25891 max_load_reduction_pct=0.438920 moved_to_replicas=147815 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[50913, 44241, 30608, 29223, 47040, 11147, 22660, 26312] avg=32768.0000 max=50913 max_rank=0 min=11147 min_rank=5 imbalance_ratio=1.553741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[50913, 44241, 30608, 29223, 47040, 11147, 22660, 26312] avg=32768.0000 max=50913 max_rank=0 min=11147 min_rank=5 imbalance_ratio=1.553741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] avg=32768.0000 max=33097 max_rank=7 min=30471 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25891 max_load_reduction_pct=0.438920 moved_to_replicas=147815 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] avg=32768.0000 max=33096 max_rank=0 min=30608 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17817 max_load_reduction_pct=0.349950 moved_to_replicas=99788 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] avg=32768.0000 max=33097 max_rank=7 min=30471 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25891 max_load_reduction_pct=0.438920 moved_to_replicas=147815 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 33096, 30471, 33097] avg=32768.0000 max=33097 max_rank=7 min=30471 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25891 max_load_reduction_pct=0.438920 moved_to_replicas=147815 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] avg=32768.0000 max=33096 max_rank=0 min=30608 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17817 max_load_reduction_pct=0.349950 moved_to_replicas=99788 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[35674, 11577, 69473, 18156, 72896, 6965, 21005, 26398] avg=32768.0000 max=72896 max_rank=4 min=6965 min_rank=5 imbalance_ratio=2.224609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] avg=32768.0000 max=33096 max_rank=0 min=30608 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17817 max_load_reduction_pct=0.349950 moved_to_replicas=99788 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[50913, 44241, 30608, 29223, 47040, 11147, 22660, 26312] avg=32768.0000 max=50913 max_rank=0 min=11147 min_rank=5 imbalance_ratio=1.553741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[50913, 44241, 30608, 29223, 47040, 11147, 22660, 26312] avg=32768.0000 max=50913 max_rank=0 min=11147 min_rank=5 imbalance_ratio=1.553741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[35674, 11577, 69473, 18156, 72896, 6965, 21005, 26398] avg=32768.0000 max=72896 max_rank=4 min=6965 min_rank=5 imbalance_ratio=2.224609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39799 max_load_reduction_pct=0.545970 moved_to_replicas=156986 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[50913, 44241, 30608, 29223, 47040, 11147, 22660, 26312] avg=32768.0000 max=50913 max_rank=0 min=11147 min_rank=5 imbalance_ratio=1.553741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[50913, 44241, 30608, 29223, 47040, 11147, 22660, 26312] avg=32768.0000 max=50913 max_rank=0 min=11147 min_rank=5 imbalance_ratio=1.553741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] avg=32768.0000 max=33096 max_rank=0 min=30608 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17817 max_load_reduction_pct=0.349950 moved_to_replicas=99788 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[35674, 11577, 69473, 18156, 72896, 6965, 21005, 26398] avg=32768.0000 max=72896 max_rank=4 min=6965 min_rank=5 imbalance_ratio=2.224609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[35674, 11577, 69473, 18156, 72896, 6965, 21005, 26398] avg=32768.0000 max=72896 max_rank=4 min=6965 min_rank=5 imbalance_ratio=2.224609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] avg=32768.0000 max=33096 max_rank=0 min=30608 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17817 max_load_reduction_pct=0.349950 moved_to_replicas=99788 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39799 max_load_reduction_pct=0.545970 moved_to_replicas=156986 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] avg=32768.0000 max=33096 max_rank=0 min=30608 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17817 max_load_reduction_pct=0.349950 moved_to_replicas=99788 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[33096, 33096, 30608, 33096, 33096, 33095, 33096, 32961] avg=32768.0000 max=33096 max_rank=0 min=30608 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=17817 max_load_reduction_pct=0.349950 moved_to_replicas=99788 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[38276, 27297, 38561, 22228, 26868, 35436, 22536, 50942] avg=32768.0000 max=50942 max_rank=7 min=22228 min_rank=3 imbalance_ratio=1.554626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39799 max_load_reduction_pct=0.545970 moved_to_replicas=156986 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39799 max_load_reduction_pct=0.545970 moved_to_replicas=156986 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[35674, 11577, 69473, 18156, 72896, 6965, 21005, 26398] avg=32768.0000 max=72896 max_rank=4 min=6965 min_rank=5 imbalance_ratio=2.224609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[35674, 11577, 69473, 18156, 72896, 6965, 21005, 26398] avg=32768.0000 max=72896 max_rank=4 min=6965 min_rank=5 imbalance_ratio=2.224609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] avg=32768.0000 max=33097 max_rank=5 min=31647 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17845 max_load_reduction_pct=0.350300 moved_to_replicas=50198 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[35674, 11577, 69473, 18156, 72896, 6965, 21005, 26398] avg=32768.0000 max=72896 max_rank=4 min=6965 min_rank=5 imbalance_ratio=2.224609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[35674, 11577, 69473, 18156, 72896, 6965, 21005, 26398] avg=32768.0000 max=72896 max_rank=4 min=6965 min_rank=5 imbalance_ratio=2.224609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[38276, 27297, 38561, 22228, 26868, 35436, 22536, 50942] avg=32768.0000 max=50942 max_rank=7 min=22228 min_rank=3 imbalance_ratio=1.554626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39799 max_load_reduction_pct=0.545970 moved_to_replicas=156986 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[38276, 27297, 38561, 22228, 26868, 35436, 22536, 50942] avg=32768.0000 max=50942 max_rank=7 min=22228 min_rank=3 imbalance_ratio=1.554626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[38276, 27297, 38561, 22228, 26868, 35436, 22536, 50942] avg=32768.0000 max=50942 max_rank=7 min=22228 min_rank=3 imbalance_ratio=1.554626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39799 max_load_reduction_pct=0.545970 moved_to_replicas=156986 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39799 max_load_reduction_pct=0.545970 moved_to_replicas=156986 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 33096, 33095, 33096, 30472] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=39799 max_load_reduction_pct=0.545970 moved_to_replicas=156986 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] avg=32768.0000 max=33097 max_rank=5 min=31647 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17845 max_load_reduction_pct=0.350300 moved_to_replicas=50198 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] avg=32768.0000 max=33097 max_rank=5 min=31647 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17845 max_load_reduction_pct=0.350300 moved_to_replicas=50198 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] avg=32768.0000 max=33097 max_rank=5 min=31647 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17845 max_load_reduction_pct=0.350300 moved_to_replicas=50198 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[16478, 69936, 32899, 43540, 45271, 17744, 13205, 23071] avg=32768.0000 max=69936 max_rank=1 min=13205 min_rank=6 imbalance_ratio=2.134277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[38276, 27297, 38561, 22228, 26868, 35436, 22536, 50942] avg=32768.0000 max=50942 max_rank=7 min=22228 min_rank=3 imbalance_ratio=1.554626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[38276, 27297, 38561, 22228, 26868, 35436, 22536, 50942] avg=32768.0000 max=50942 max_rank=7 min=22228 min_rank=3 imbalance_ratio=1.554626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[38276, 27297, 38561, 22228, 26868, 35436, 22536, 50942] avg=32768.0000 max=50942 max_rank=7 min=22228 min_rank=3 imbalance_ratio=1.554626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[38276, 27297, 38561, 22228, 26868, 35436, 22536, 50942] avg=32768.0000 max=50942 max_rank=7 min=22228 min_rank=3 imbalance_ratio=1.554626 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30669 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=36839 max_load_reduction_pct=0.526753 moved_to_replicas=116389 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[16478, 69936, 32899, 43540, 45271, 17744, 13205, 23071] avg=32768.0000 max=69936 max_rank=1 min=13205 min_rank=6 imbalance_ratio=2.134277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] avg=32768.0000 max=33097 max_rank=5 min=31647 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17845 max_load_reduction_pct=0.350300 moved_to_replicas=50198 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[16478, 69936, 32899, 43540, 45271, 17744, 13205, 23071] avg=32768.0000 max=69936 max_rank=1 min=13205 min_rank=6 imbalance_ratio=2.134277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] avg=32768.0000 max=33097 max_rank=5 min=31647 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17845 max_load_reduction_pct=0.350300 moved_to_replicas=50198 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[16478, 69936, 32899, 43540, 45271, 17744, 13205, 23071] avg=32768.0000 max=69936 max_rank=1 min=13205 min_rank=6 imbalance_ratio=2.134277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] avg=32768.0000 max=33097 max_rank=5 min=31647 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17845 max_load_reduction_pct=0.350300 moved_to_replicas=50198 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[33095, 32478, 33096, 33096, 32539, 33097, 31647, 33096] avg=32768.0000 max=33097 max_rank=5 min=31647 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=17845 max_load_reduction_pct=0.350300 moved_to_replicas=50198 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30669 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=36839 max_load_reduction_pct=0.526753 moved_to_replicas=116389 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[48930, 21595, 8342, 37564, 25037, 43589, 34318, 42769] avg=32768.0000 max=48930 max_rank=0 min=8342 min_rank=2 imbalance_ratio=1.493225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30669 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=36839 max_load_reduction_pct=0.526753 moved_to_replicas=116389 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30669 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=36839 max_load_reduction_pct=0.526753 moved_to_replicas=116389 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[16478, 69936, 32899, 43540, 45271, 17744, 13205, 23071] avg=32768.0000 max=69936 max_rank=1 min=13205 min_rank=6 imbalance_ratio=2.134277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[16478, 69936, 32899, 43540, 45271, 17744, 13205, 23071] avg=32768.0000 max=69936 max_rank=1 min=13205 min_rank=6 imbalance_ratio=2.134277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[16478, 69936, 32899, 43540, 45271, 17744, 13205, 23071] avg=32768.0000 max=69936 max_rank=1 min=13205 min_rank=6 imbalance_ratio=2.134277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31480 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15833 max_load_reduction_pct=0.323585 moved_to_replicas=97276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[16478, 69936, 32899, 43540, 45271, 17744, 13205, 23071] avg=32768.0000 max=69936 max_rank=1 min=13205 min_rank=6 imbalance_ratio=2.134277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[48930, 21595, 8342, 37564, 25037, 43589, 34318, 42769] avg=32768.0000 max=48930 max_rank=0 min=8342 min_rank=2 imbalance_ratio=1.493225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30669 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=36839 max_load_reduction_pct=0.526753 moved_to_replicas=116389 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[48930, 21595, 8342, 37564, 25037, 43589, 34318, 42769] avg=32768.0000 max=48930 max_rank=0 min=8342 min_rank=2 imbalance_ratio=1.493225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[48930, 21595, 8342, 37564, 25037, 43589, 34318, 42769] avg=32768.0000 max=48930 max_rank=0 min=8342 min_rank=2 imbalance_ratio=1.493225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30669 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=36839 max_load_reduction_pct=0.526753 moved_to_replicas=116389 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30669 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=36839 max_load_reduction_pct=0.526753 moved_to_replicas=116389 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[33096, 33096, 32899, 33096, 33095, 30669, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=30669 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=36839 max_load_reduction_pct=0.526753 moved_to_replicas=116389 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31480 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15833 max_load_reduction_pct=0.323585 moved_to_replicas=97276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31480 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15833 max_load_reduction_pct=0.323585 moved_to_replicas=97276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31480 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15833 max_load_reduction_pct=0.323585 moved_to_replicas=97276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[48930, 21595, 8342, 37564, 25037, 43589, 34318, 42769] avg=32768.0000 max=48930 max_rank=0 min=8342 min_rank=2 imbalance_ratio=1.493225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[48930, 21595, 8342, 37564, 25037, 43589, 34318, 42769] avg=32768.0000 max=48930 max_rank=0 min=8342 min_rank=2 imbalance_ratio=1.493225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[48930, 21595, 8342, 37564, 25037, 43589, 34318, 42769] avg=32768.0000 max=48930 max_rank=0 min=8342 min_rank=2 imbalance_ratio=1.493225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[48930, 21595, 8342, 37564, 25037, 43589, 34318, 42769] avg=32768.0000 max=48930 max_rank=0 min=8342 min_rank=2 imbalance_ratio=1.493225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31480 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15833 max_load_reduction_pct=0.323585 moved_to_replicas=97276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31480 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15833 max_load_reduction_pct=0.323585 moved_to_replicas=97276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31480 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15833 max_load_reduction_pct=0.323585 moved_to_replicas=97276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[33096, 32088, 33097, 33096, 31480, 33096, 33096, 33095] avg=32768.0000 max=33097 max_rank=2 min=31480 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15833 max_load_reduction_pct=0.323585 moved_to_replicas=97276 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=32 total_expected_recv=1047457 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=252800 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=32 total_expected_recv=1047482 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253056 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=32 total_expected_recv=1053025 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=247552 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=32 total_expected_recv=1050291 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=250368 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=32 total_expected_recv=1047634 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=252416 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=32 total_expected_recv=1046435 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254080 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=32 total_expected_recv=1047283 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253824 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=32 total_expected_recv=1049001 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=251520 peak_demand=33097 required_capacity=1.010040 + [2026-08-30 05:10:26] iteration 2/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 1120.5 | throughput per GPU (TFLOP/s/GPU): 297.5 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101621E+01 | loss scale: 1.0 | grad norm: 45.876 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=40960 expected_recv=65832 expected_overflow=0 exact_recv_per_rank=[32971, 33096, 31896, 33097, 33097, 33095, 31796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=40960 expected_recv=66067 expected_overflow=0 exact_recv_per_rank=[32971, 33096, 31896, 33097, 33097, 33095, 31796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=40960 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[32971, 33096, 31896, 33097, 33097, 33095, 31796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[32971, 33096, 31896, 33097, 33097, 33095, 31796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=40960 expected_recv=63971 expected_overflow=0 exact_recv_per_rank=[32971, 33096, 31896, 33097, 33097, 33095, 31796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=40960 expected_recv=63648 expected_overflow=0 exact_recv_per_rank=[32971, 33096, 31896, 33097, 33097, 33095, 31796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32971, 33096, 31896, 33097, 33097, 33095, 31796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=2 budget=40960 expected_recv=65832 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=2 budget=40960 expected_recv=66067 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=2 budget=40960 expected_recv=66190 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=2 budget=40960 expected_recv=63971 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=2 budget=40960 expected_recv=63648 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[32971, 33096, 31896, 33097, 33097, 33095, 31796, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=40960 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[33097, 32445, 31125, 33094, 33096, 33095, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33097, 32445, 31125, 33094, 33096, 33095, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33097, 32445, 31125, 33094, 33096, 33095, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=40960 expected_recv=65499 expected_overflow=0 exact_recv_per_rank=[33097, 32445, 31125, 33094, 33096, 33095, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=2 budget=40960 expected_recv=65541 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=40960 expected_recv=62288 expected_overflow=0 exact_recv_per_rank=[33097, 32445, 31125, 33094, 33096, 33095, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=2 budget=40960 expected_recv=65499 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33097, 32445, 31125, 33094, 33096, 33095, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33097, 32445, 31125, 33094, 33096, 33095, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=2 budget=40960 expected_recv=62288 actual_valid_recv=63104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=63104 slack_slots=18816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32873, 31607, 32184, 33096, 33096, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=40960 expected_recv=64702 expected_overflow=0 exact_recv_per_rank=[32873, 31607, 32184, 33096, 33096, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=40960 expected_recv=65969 expected_overflow=0 exact_recv_per_rank=[32873, 31607, 32184, 33096, 33096, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 32445, 31125, 33094, 33096, 33095, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=2 budget=40960 expected_recv=64702 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32873, 31607, 32184, 33096, 33096, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=2 budget=40960 expected_recv=65969 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=40960 expected_recv=64697 expected_overflow=0 exact_recv_per_rank=[32873, 31607, 32184, 33096, 33096, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[32873, 31607, 32184, 33096, 33096, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=40960 expected_recv=65892 expected_overflow=0 exact_recv_per_rank=[32873, 31607, 32184, 33096, 33096, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=2 budget=40960 expected_recv=64697 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=2 budget=40960 expected_recv=65892 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=40960 expected_recv=65957 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30730, 32862, 33072, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30730, 32862, 33072, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30730, 32862, 33072, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=2 budget=40960 expected_recv=65957 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=40960 expected_recv=64452 expected_overflow=0 exact_recv_per_rank=[32873, 31607, 32184, 33096, 33096, 33096, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30730, 32862, 33072, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=2 budget=40960 expected_recv=64452 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30730, 32862, 33072, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=40960 expected_recv=63544 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30730, 32862, 33072, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30730, 32862, 33072, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=2 budget=40960 expected_recv=63544 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=40960 expected_recv=65699 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30964, 33096, 33097, 32603, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30964, 33096, 33097, 32603, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=2 budget=40960 expected_recv=65699 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30964, 33096, 33097, 32603, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30964, 33096, 33097, 32603, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=40960 expected_recv=63826 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 30730, 32862, 33072, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=40960 expected_recv=62078 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30964, 33096, 33097, 32603, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=2 budget=40960 expected_recv=63826 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30964, 33096, 33097, 32603, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30964, 33096, 33097, 32603, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=2 budget=40960 expected_recv=62078 actual_valid_recv=62976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=62976 slack_slots=18944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=40960 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[32141, 33094, 33096, 31427, 33098, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=40960 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[32141, 33094, 33096, 31427, 33098, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=2 budget=40960 expected_recv=66190 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=40960 expected_recv=64257 expected_overflow=0 exact_recv_per_rank=[32141, 33094, 33096, 31427, 33098, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=2 budget=40960 expected_recv=66190 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=40960 expected_recv=62879 expected_overflow=0 exact_recv_per_rank=[32141, 33094, 33096, 31427, 33098, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=2 budget=40960 expected_recv=64257 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=40960 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30964, 33096, 33097, 32603, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=2 budget=40960 expected_recv=62879 actual_valid_recv=63488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32141, 33094, 33096, 31427, 33098, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[32141, 33094, 33096, 31427, 33098, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=2 budget=40960 expected_recv=65550 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=63488 slack_slots=18432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32141, 33094, 33096, 31427, 33098, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32307, 33096, 33097, 33095, 33096, 33096, 32147, 32210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32307, 33096, 33097, 33095, 33096, 33096, 32147, 32210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=40960 expected_recv=64517 expected_overflow=0 exact_recv_per_rank=[32307, 33096, 33097, 33095, 33096, 33096, 32147, 32210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=2 budget=40960 expected_recv=64517 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=40960 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[32307, 33096, 33097, 33095, 33096, 33096, 32147, 32210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[32141, 33094, 33096, 31427, 33098, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=2 budget=40960 expected_recv=66190 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=40960 expected_recv=64887 expected_overflow=0 exact_recv_per_rank=[32307, 33096, 33097, 33095, 33096, 33096, 32147, 32210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=40960 expected_recv=65243 expected_overflow=0 exact_recv_per_rank=[32307, 33096, 33097, 33095, 33096, 33096, 32147, 32210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=40960 expected_recv=64874 expected_overflow=0 exact_recv_per_rank=[32307, 33096, 33097, 33095, 33096, 33096, 32147, 32210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30674, 33096, 33096, 32894, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=2 budget=40960 expected_recv=64887 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=2 budget=40960 expected_recv=65243 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=2 budget=40960 expected_recv=64874 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30674, 33096, 33096, 32894, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=40960 expected_recv=63518 expected_overflow=0 exact_recv_per_rank=[30674, 33096, 33096, 32894, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=2 budget=40960 expected_recv=63518 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=40960 expected_recv=63955 expected_overflow=0 exact_recv_per_rank=[30674, 33096, 33096, 32894, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32307, 33096, 33097, 33095, 33096, 33096, 32147, 32210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=2 budget=40960 expected_recv=63955 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=40960 expected_recv=65855 expected_overflow=0 exact_recv_per_rank=[30674, 33096, 33096, 32894, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30674, 33096, 33096, 32894, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30674, 33096, 33096, 32894, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 30472, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=2 budget=40960 expected_recv=65855 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 30472, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67456 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 30472, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=67456 slack_slots=14464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=40960 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 30472, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[30674, 33096, 33096, 32894, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=2 budget=40960 expected_recv=63568 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 30472, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32184, 33097, 31618, 33097, 33095, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 30472, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=40960 expected_recv=63566 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 30472, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=40960 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32184, 33097, 31618, 33097, 33095, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=2 budget=40960 expected_recv=63566 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32184, 33097, 31618, 33097, 33095, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=2 budget=40960 expected_recv=66190 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=40960 expected_recv=64891 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32184, 33097, 31618, 33097, 33095, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33095, 30472, 33095, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=2 budget=40960 expected_recv=64891 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=40960 expected_recv=64684 expected_overflow=0 exact_recv_per_rank=[33096, 31407, 33096, 33096, 33096, 32161, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=40960 expected_recv=65280 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32184, 33097, 31618, 33097, 33095, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=40960 expected_recv=65958 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32184, 33097, 31618, 33097, 33095, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=40960 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32184, 33097, 31618, 33097, 33095, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=2 budget=40960 expected_recv=64684 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=40960 expected_recv=63017 expected_overflow=0 exact_recv_per_rank=[33096, 31407, 33096, 33096, 33096, 32161, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=2 budget=40960 expected_recv=65280 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=2 budget=40960 expected_recv=65958 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=2 budget=40960 expected_recv=66190 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=40960 expected_recv=65626 expected_overflow=0 exact_recv_per_rank=[33096, 31407, 33096, 33096, 33096, 32161, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=2 budget=40960 expected_recv=63017 actual_valid_recv=64000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31407, 33096, 33096, 33096, 32161, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=64000 slack_slots=17920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=2 budget=40960 expected_recv=65626 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=40960 expected_recv=63393 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32184, 33097, 31618, 33097, 33095, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=2 budget=40960 expected_recv=63393 actual_valid_recv=64128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=40960 expected_recv=64222 expected_overflow=0 exact_recv_per_rank=[31616, 33096, 33096, 33096, 33096, 32880, 32168, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=64128 slack_slots=17792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31407, 33096, 33096, 33096, 32161, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 31407, 33096, 33096, 33096, 32161, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=2 budget=40960 expected_recv=64222 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31407, 33096, 33096, 33096, 32161, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31616, 33096, 33096, 33096, 33096, 32880, 32168, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=40960 expected_recv=64713 expected_overflow=0 exact_recv_per_rank=[31616, 33096, 33096, 33096, 33096, 32880, 32168, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[31616, 33096, 33096, 33096, 33096, 32880, 32168, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31407, 33096, 33096, 33096, 32161, 33097, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=2 budget=40960 expected_recv=64713 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=40960 expected_recv=65654 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 33094, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=40960 expected_recv=65321 expected_overflow=0 exact_recv_per_rank=[31616, 33096, 33096, 33096, 33096, 32880, 32168, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=2 budget=40960 expected_recv=65654 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31616, 33096, 33096, 33096, 33096, 32880, 32168, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=40960 expected_recv=65264 expected_overflow=0 exact_recv_per_rank=[31616, 33096, 33096, 33096, 33096, 32880, 32168, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 33094, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=2 budget=40960 expected_recv=65321 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=2 budget=40960 expected_recv=65264 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=12 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=40960 expected_recv=65896 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 33094, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 33094, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31616, 33096, 33096, 33096, 33096, 32880, 32168, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=12 calls=2 budget=40960 expected_recv=65896 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32916, 33096, 33096, 30653, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 33094, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 33094, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=12 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=40960 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 33094, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32916, 33096, 33096, 30653, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=40960 expected_recv=66012 expected_overflow=0 exact_recv_per_rank=[32916, 33096, 33096, 30653, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=12 calls=2 budget=40960 expected_recv=66190 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=40960 expected_recv=63749 expected_overflow=0 exact_recv_per_rank=[32916, 33096, 33096, 30653, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=40960 expected_recv=61778 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33097, 33094, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=2 budget=40960 expected_recv=66012 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 30812, 33096, 32756, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=12 calls=2 budget=40960 expected_recv=61778 actual_valid_recv=62848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=2 budget=40960 expected_recv=63749 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=62848 slack_slots=19072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32916, 33096, 33096, 30653, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32916, 33096, 33096, 30653, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=13 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=40960 expected_recv=63568 expected_overflow=0 exact_recv_per_rank=[32916, 33096, 33096, 30653, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 30812, 33096, 32756, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 30812, 33096, 32756, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=13 calls=2 budget=40960 expected_recv=63568 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=14 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32916, 33096, 33096, 30653, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 30812, 33096, 32756, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65726 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32627, 33096, 32729, 33096, 31308, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65726 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 30812, 33096, 32756, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=40960 expected_recv=65920 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 30812, 33096, 32756, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=14 calls=2 budget=40960 expected_recv=65920 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32627, 33096, 32729, 33096, 31308, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=40960 expected_recv=65853 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 30812, 33096, 32756, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32627, 33096, 32729, 33096, 31308, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=40960 expected_recv=61556 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 33096, 30812, 33096, 32756, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=15 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=14 calls=2 budget=40960 expected_recv=65853 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65941 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32627, 33096, 32729, 33096, 31308, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=2 budget=40960 expected_recv=61556 actual_valid_recv=62336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[32971, 30631, 31223, 17802, 38478, 48396, 26409, 36234] avg=32768.0000 max=48396 max_rank=5 min=17802 min_rank=3 imbalance_ratio=1.476929 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65941 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32627, 33096, 32729, 33096, 31308, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=62336 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=40960 expected_recv=64512 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32627, 33096, 32729, 33096, 31308, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=15 calls=2 budget=40960 expected_recv=64512 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[32971, 30631, 31223, 17802, 38478, 48396, 26409, 36234] avg=32768.0000 max=48396 max_rank=5 min=17802 min_rank=3 imbalance_ratio=1.476929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[32971, 33096, 31896, 33097, 33097, 33095, 31796, 33096] avg=32768.0000 max=33097 max_rank=3 min=31796 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=15299 max_load_reduction_pct=0.316121 moved_to_replicas=45376 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=40960 expected_recv=63999 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32627, 33096, 32729, 33096, 31308, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[32971, 30631, 31223, 17802, 38478, 48396, 26409, 36234] avg=32768.0000 max=48396 max_rank=5 min=17802 min_rank=3 imbalance_ratio=1.476929 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32627, 33096, 32729, 33096, 31308, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=15 calls=2 budget=40960 expected_recv=63999 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[32971, 30631, 31223, 17802, 38478, 48396, 26409, 36234] avg=32768.0000 max=48396 max_rank=5 min=17802 min_rank=3 imbalance_ratio=1.476929 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=2 budget=40960 expected_recv=65534 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[32971, 33096, 31896, 33097, 33097, 33095, 31796, 33096] avg=32768.0000 max=33097 max_rank=3 min=31796 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=15299 max_load_reduction_pct=0.316121 moved_to_replicas=45376 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[32971, 30631, 31223, 17802, 38478, 48396, 26409, 36234] avg=32768.0000 max=48396 max_rank=5 min=17802 min_rank=3 imbalance_ratio=1.476929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[42807, 14582, 244, 253, 2471, 36970, 88392, 76425] avg=32768.0000 max=88392 max_rank=6 min=244 min_rank=2 imbalance_ratio=2.697510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[32971, 30631, 31223, 17802, 38478, 48396, 26409, 36234] avg=32768.0000 max=48396 max_rank=5 min=17802 min_rank=3 imbalance_ratio=1.476929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[32971, 33096, 31896, 33097, 33097, 33095, 31796, 33096] avg=32768.0000 max=33097 max_rank=3 min=31796 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=15299 max_load_reduction_pct=0.316121 moved_to_replicas=45376 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[32971, 33096, 31896, 33097, 33097, 33095, 31796, 33096] avg=32768.0000 max=33097 max_rank=3 min=31796 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=15299 max_load_reduction_pct=0.316121 moved_to_replicas=45376 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32445, 31125, 33094, 33096, 33095, 33097, 33095] avg=32768.0000 max=33097 max_rank=0 min=31125 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55295 max_load_reduction_pct=0.625566 moved_to_replicas=214974 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[32971, 33096, 31896, 33097, 33097, 33095, 31796, 33096] avg=32768.0000 max=33097 max_rank=3 min=31796 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=15299 max_load_reduction_pct=0.316121 moved_to_replicas=45376 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[32971, 30631, 31223, 17802, 38478, 48396, 26409, 36234] avg=32768.0000 max=48396 max_rank=5 min=17802 min_rank=3 imbalance_ratio=1.476929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[42807, 14582, 244, 253, 2471, 36970, 88392, 76425] avg=32768.0000 max=88392 max_rank=6 min=244 min_rank=2 imbalance_ratio=2.697510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[32971, 33096, 31896, 33097, 33097, 33095, 31796, 33096] avg=32768.0000 max=33097 max_rank=3 min=31796 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=15299 max_load_reduction_pct=0.316121 moved_to_replicas=45376 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[32971, 30631, 31223, 17802, 38478, 48396, 26409, 36234] avg=32768.0000 max=48396 max_rank=5 min=17802 min_rank=3 imbalance_ratio=1.476929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[42807, 14582, 244, 253, 2471, 36970, 88392, 76425] avg=32768.0000 max=88392 max_rank=6 min=244 min_rank=2 imbalance_ratio=2.697510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[42807, 14582, 244, 253, 2471, 36970, 88392, 76425] avg=32768.0000 max=88392 max_rank=6 min=244 min_rank=2 imbalance_ratio=2.697510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32445, 31125, 33094, 33096, 33095, 33097, 33095] avg=32768.0000 max=33097 max_rank=0 min=31125 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55295 max_load_reduction_pct=0.625566 moved_to_replicas=214974 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[32971, 33096, 31896, 33097, 33097, 33095, 31796, 33096] avg=32768.0000 max=33097 max_rank=3 min=31796 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=15299 max_load_reduction_pct=0.316121 moved_to_replicas=45376 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[30983, 31607, 32184, 41572, 30265, 48414, 45111, 2008] avg=32768.0000 max=48414 max_rank=5 min=2008 min_rank=7 imbalance_ratio=1.477478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32445, 31125, 33094, 33096, 33095, 33097, 33095] avg=32768.0000 max=33097 max_rank=0 min=31125 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55295 max_load_reduction_pct=0.625566 moved_to_replicas=214974 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[32971, 33096, 31896, 33097, 33097, 33095, 31796, 33096] avg=32768.0000 max=33097 max_rank=3 min=31796 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=15299 max_load_reduction_pct=0.316121 moved_to_replicas=45376 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[42807, 14582, 244, 253, 2471, 36970, 88392, 76425] avg=32768.0000 max=88392 max_rank=6 min=244 min_rank=2 imbalance_ratio=2.697510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[42807, 14582, 244, 253, 2471, 36970, 88392, 76425] avg=32768.0000 max=88392 max_rank=6 min=244 min_rank=2 imbalance_ratio=2.697510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32445, 31125, 33094, 33096, 33095, 33097, 33095] avg=32768.0000 max=33097 max_rank=0 min=31125 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55295 max_load_reduction_pct=0.625566 moved_to_replicas=214974 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[32873, 31607, 32184, 33096, 33096, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31607 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=15317 max_load_reduction_pct=0.316375 moved_to_replicas=68996 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[30983, 31607, 32184, 41572, 30265, 48414, 45111, 2008] avg=32768.0000 max=48414 max_rank=5 min=2008 min_rank=7 imbalance_ratio=1.477478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32445, 31125, 33094, 33096, 33095, 33097, 33095] avg=32768.0000 max=33097 max_rank=0 min=31125 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55295 max_load_reduction_pct=0.625566 moved_to_replicas=214974 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32445, 31125, 33094, 33096, 33095, 33097, 33095] avg=32768.0000 max=33097 max_rank=0 min=31125 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55295 max_load_reduction_pct=0.625566 moved_to_replicas=214974 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[42807, 14582, 244, 253, 2471, 36970, 88392, 76425] avg=32768.0000 max=88392 max_rank=6 min=244 min_rank=2 imbalance_ratio=2.697510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[30983, 31607, 32184, 41572, 30265, 48414, 45111, 2008] avg=32768.0000 max=48414 max_rank=5 min=2008 min_rank=7 imbalance_ratio=1.477478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[42807, 14582, 244, 253, 2471, 36970, 88392, 76425] avg=32768.0000 max=88392 max_rank=6 min=244 min_rank=2 imbalance_ratio=2.697510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[30983, 31607, 32184, 41572, 30265, 48414, 45111, 2008] avg=32768.0000 max=48414 max_rank=5 min=2008 min_rank=7 imbalance_ratio=1.477478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[32873, 31607, 32184, 33096, 33096, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31607 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=15317 max_load_reduction_pct=0.316375 moved_to_replicas=68996 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[44629, 29705, 44283, 39017, 765, 526, 30542, 72677] avg=32768.0000 max=72677 max_rank=7 min=526 min_rank=5 imbalance_ratio=2.217926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32445, 31125, 33094, 33096, 33095, 33097, 33095] avg=32768.0000 max=33097 max_rank=0 min=31125 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55295 max_load_reduction_pct=0.625566 moved_to_replicas=214974 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[32873, 31607, 32184, 33096, 33096, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31607 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=15317 max_load_reduction_pct=0.316375 moved_to_replicas=68996 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[33097, 32445, 31125, 33094, 33096, 33095, 33097, 33095] avg=32768.0000 max=33097 max_rank=0 min=31125 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=55295 max_load_reduction_pct=0.625566 moved_to_replicas=214974 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[30983, 31607, 32184, 41572, 30265, 48414, 45111, 2008] avg=32768.0000 max=48414 max_rank=5 min=2008 min_rank=7 imbalance_ratio=1.477478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[30983, 31607, 32184, 41572, 30265, 48414, 45111, 2008] avg=32768.0000 max=48414 max_rank=5 min=2008 min_rank=7 imbalance_ratio=1.477478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[32873, 31607, 32184, 33096, 33096, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31607 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=15317 max_load_reduction_pct=0.316375 moved_to_replicas=68996 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30730, 32862, 33072, 33096] avg=32768.0000 max=33096 max_rank=0 min=30730 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=39581 max_load_reduction_pct=0.544615 moved_to_replicas=143097 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[44629, 29705, 44283, 39017, 765, 526, 30542, 72677] avg=32768.0000 max=72677 max_rank=7 min=526 min_rank=5 imbalance_ratio=2.217926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[32873, 31607, 32184, 33096, 33096, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31607 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=15317 max_load_reduction_pct=0.316375 moved_to_replicas=68996 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[32873, 31607, 32184, 33096, 33096, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31607 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=15317 max_load_reduction_pct=0.316375 moved_to_replicas=68996 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[30983, 31607, 32184, 41572, 30265, 48414, 45111, 2008] avg=32768.0000 max=48414 max_rank=5 min=2008 min_rank=7 imbalance_ratio=1.477478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[44629, 29705, 44283, 39017, 765, 526, 30542, 72677] avg=32768.0000 max=72677 max_rank=7 min=526 min_rank=5 imbalance_ratio=2.217926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[30983, 31607, 32184, 41572, 30265, 48414, 45111, 2008] avg=32768.0000 max=48414 max_rank=5 min=2008 min_rank=7 imbalance_ratio=1.477478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[44629, 29705, 44283, 39017, 765, 526, 30542, 72677] avg=32768.0000 max=72677 max_rank=7 min=526 min_rank=5 imbalance_ratio=2.217926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30730, 32862, 33072, 33096] avg=32768.0000 max=33096 max_rank=0 min=30730 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=39581 max_load_reduction_pct=0.544615 moved_to_replicas=143097 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[4476, 8166, 20220, 95933, 7301, 32603, 54520, 38925] avg=32768.0000 max=95933 max_rank=3 min=4476 min_rank=0 imbalance_ratio=2.927643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[32873, 31607, 32184, 33096, 33096, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31607 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=15317 max_load_reduction_pct=0.316375 moved_to_replicas=68996 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30730, 32862, 33072, 33096] avg=32768.0000 max=33096 max_rank=0 min=30730 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=39581 max_load_reduction_pct=0.544615 moved_to_replicas=143097 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[32873, 31607, 32184, 33096, 33096, 33096, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31607 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=15317 max_load_reduction_pct=0.316375 moved_to_replicas=68996 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[44629, 29705, 44283, 39017, 765, 526, 30542, 72677] avg=32768.0000 max=72677 max_rank=7 min=526 min_rank=5 imbalance_ratio=2.217926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[44629, 29705, 44283, 39017, 765, 526, 30542, 72677] avg=32768.0000 max=72677 max_rank=7 min=526 min_rank=5 imbalance_ratio=2.217926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30730, 32862, 33072, 33096] avg=32768.0000 max=33096 max_rank=0 min=30730 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=39581 max_load_reduction_pct=0.544615 moved_to_replicas=143097 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[33096, 33096, 30964, 33096, 33097, 32603, 33095, 33097] avg=32768.0000 max=33097 max_rank=4 min=30964 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62836 max_load_reduction_pct=0.654999 moved_to_replicas=175611 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[4476, 8166, 20220, 95933, 7301, 32603, 54520, 38925] avg=32768.0000 max=95933 max_rank=3 min=4476 min_rank=0 imbalance_ratio=2.927643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30730, 32862, 33072, 33096] avg=32768.0000 max=33096 max_rank=0 min=30730 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=39581 max_load_reduction_pct=0.544615 moved_to_replicas=143097 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30730, 32862, 33072, 33096] avg=32768.0000 max=33096 max_rank=0 min=30730 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=39581 max_load_reduction_pct=0.544615 moved_to_replicas=143097 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[44629, 29705, 44283, 39017, 765, 526, 30542, 72677] avg=32768.0000 max=72677 max_rank=7 min=526 min_rank=5 imbalance_ratio=2.217926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[44629, 29705, 44283, 39017, 765, 526, 30542, 72677] avg=32768.0000 max=72677 max_rank=7 min=526 min_rank=5 imbalance_ratio=2.217926 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[4476, 8166, 20220, 95933, 7301, 32603, 54520, 38925] avg=32768.0000 max=95933 max_rank=3 min=4476 min_rank=0 imbalance_ratio=2.927643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[4476, 8166, 20220, 95933, 7301, 32603, 54520, 38925] avg=32768.0000 max=95933 max_rank=3 min=4476 min_rank=0 imbalance_ratio=2.927643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[28335, 4345, 34364, 1692, 70845, 18092, 45221, 59250] avg=32768.0000 max=70845 max_rank=4 min=1692 min_rank=3 imbalance_ratio=2.162018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[33096, 33096, 30964, 33096, 33097, 32603, 33095, 33097] avg=32768.0000 max=33097 max_rank=4 min=30964 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62836 max_load_reduction_pct=0.654999 moved_to_replicas=175611 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30730, 32862, 33072, 33096] avg=32768.0000 max=33096 max_rank=0 min=30730 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=39581 max_load_reduction_pct=0.544615 moved_to_replicas=143097 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 30730, 32862, 33072, 33096] avg=32768.0000 max=33096 max_rank=0 min=30730 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=39581 max_load_reduction_pct=0.544615 moved_to_replicas=143097 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[33096, 33096, 30964, 33096, 33097, 32603, 33095, 33097] avg=32768.0000 max=33097 max_rank=4 min=30964 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62836 max_load_reduction_pct=0.654999 moved_to_replicas=175611 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[4476, 8166, 20220, 95933, 7301, 32603, 54520, 38925] avg=32768.0000 max=95933 max_rank=3 min=4476 min_rank=0 imbalance_ratio=2.927643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[32141, 33094, 33096, 31427, 33098, 33095, 33097, 33096] avg=32768.0000 max=33098 max_rank=4 min=31427 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=37747 max_load_reduction_pct=0.532811 moved_to_replicas=149380 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[33096, 33096, 30964, 33096, 33097, 32603, 33095, 33097] avg=32768.0000 max=33097 max_rank=4 min=30964 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62836 max_load_reduction_pct=0.654999 moved_to_replicas=175611 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[4476, 8166, 20220, 95933, 7301, 32603, 54520, 38925] avg=32768.0000 max=95933 max_rank=3 min=4476 min_rank=0 imbalance_ratio=2.927643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[28335, 4345, 34364, 1692, 70845, 18092, 45221, 59250] avg=32768.0000 max=70845 max_rank=4 min=1692 min_rank=3 imbalance_ratio=2.162018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[33096, 33096, 30964, 33096, 33097, 32603, 33095, 33097] avg=32768.0000 max=33097 max_rank=4 min=30964 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62836 max_load_reduction_pct=0.654999 moved_to_replicas=175611 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[4476, 8166, 20220, 95933, 7301, 32603, 54520, 38925] avg=32768.0000 max=95933 max_rank=3 min=4476 min_rank=0 imbalance_ratio=2.927643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[4476, 8166, 20220, 95933, 7301, 32603, 54520, 38925] avg=32768.0000 max=95933 max_rank=3 min=4476 min_rank=0 imbalance_ratio=2.927643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[33096, 33096, 30964, 33096, 33097, 32603, 33095, 33097] avg=32768.0000 max=33097 max_rank=4 min=30964 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62836 max_load_reduction_pct=0.654999 moved_to_replicas=175611 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[28335, 4345, 34364, 1692, 70845, 18092, 45221, 59250] avg=32768.0000 max=70845 max_rank=4 min=1692 min_rank=3 imbalance_ratio=2.162018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[24134, 33589, 9478, 58220, 53756, 18610, 32147, 32210] avg=32768.0000 max=58220 max_rank=3 min=9478 min_rank=2 imbalance_ratio=1.776733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[32141, 33094, 33096, 31427, 33098, 33095, 33097, 33096] avg=32768.0000 max=33098 max_rank=4 min=31427 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=37747 max_load_reduction_pct=0.532811 moved_to_replicas=149380 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[33096, 33096, 30964, 33096, 33097, 32603, 33095, 33097] avg=32768.0000 max=33097 max_rank=4 min=30964 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62836 max_load_reduction_pct=0.654999 moved_to_replicas=175611 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[33096, 33096, 30964, 33096, 33097, 32603, 33095, 33097] avg=32768.0000 max=33097 max_rank=4 min=30964 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=62836 max_load_reduction_pct=0.654999 moved_to_replicas=175611 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[28335, 4345, 34364, 1692, 70845, 18092, 45221, 59250] avg=32768.0000 max=70845 max_rank=4 min=1692 min_rank=3 imbalance_ratio=2.162018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[28335, 4345, 34364, 1692, 70845, 18092, 45221, 59250] avg=32768.0000 max=70845 max_rank=4 min=1692 min_rank=3 imbalance_ratio=2.162018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[32141, 33094, 33096, 31427, 33098, 33095, 33097, 33096] avg=32768.0000 max=33098 max_rank=4 min=31427 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=37747 max_load_reduction_pct=0.532811 moved_to_replicas=149380 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[32307, 33096, 33097, 33095, 33096, 33096, 32147, 32210] avg=32768.0000 max=33097 max_rank=2 min=32147 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25123 max_load_reduction_pct=0.431518 moved_to_replicas=94647 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[28335, 4345, 34364, 1692, 70845, 18092, 45221, 59250] avg=32768.0000 max=70845 max_rank=4 min=1692 min_rank=3 imbalance_ratio=2.162018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[24134, 33589, 9478, 58220, 53756, 18610, 32147, 32210] avg=32768.0000 max=58220 max_rank=3 min=9478 min_rank=2 imbalance_ratio=1.776733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[32141, 33094, 33096, 31427, 33098, 33095, 33097, 33096] avg=32768.0000 max=33098 max_rank=4 min=31427 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=37747 max_load_reduction_pct=0.532811 moved_to_replicas=149380 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[32141, 33094, 33096, 31427, 33098, 33095, 33097, 33096] avg=32768.0000 max=33098 max_rank=4 min=31427 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=37747 max_load_reduction_pct=0.532811 moved_to_replicas=149380 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[28335, 4345, 34364, 1692, 70845, 18092, 45221, 59250] avg=32768.0000 max=70845 max_rank=4 min=1692 min_rank=3 imbalance_ratio=2.162018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[32141, 33094, 33096, 31427, 33098, 33095, 33097, 33096] avg=32768.0000 max=33098 max_rank=4 min=31427 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=37747 max_load_reduction_pct=0.532811 moved_to_replicas=149380 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[30491, 65524, 1392, 28822, 4467, 65592, 3795, 62061] avg=32768.0000 max=65592 max_rank=5 min=1392 min_rank=2 imbalance_ratio=2.001709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[28335, 4345, 34364, 1692, 70845, 18092, 45221, 59250] avg=32768.0000 max=70845 max_rank=4 min=1692 min_rank=3 imbalance_ratio=2.162018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[32307, 33096, 33097, 33095, 33096, 33096, 32147, 32210] avg=32768.0000 max=33097 max_rank=2 min=32147 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25123 max_load_reduction_pct=0.431518 moved_to_replicas=94647 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[24134, 33589, 9478, 58220, 53756, 18610, 32147, 32210] avg=32768.0000 max=58220 max_rank=3 min=9478 min_rank=2 imbalance_ratio=1.776733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[32141, 33094, 33096, 31427, 33098, 33095, 33097, 33096] avg=32768.0000 max=33098 max_rank=4 min=31427 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=37747 max_load_reduction_pct=0.532811 moved_to_replicas=149380 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[24134, 33589, 9478, 58220, 53756, 18610, 32147, 32210] avg=32768.0000 max=58220 max_rank=3 min=9478 min_rank=2 imbalance_ratio=1.776733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[30674, 33096, 33096, 32894, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30674 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32496 max_load_reduction_pct=0.495426 moved_to_replicas=192459 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[24134, 33589, 9478, 58220, 53756, 18610, 32147, 32210] avg=32768.0000 max=58220 max_rank=3 min=9478 min_rank=2 imbalance_ratio=1.776733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[32141, 33094, 33096, 31427, 33098, 33095, 33097, 33096] avg=32768.0000 max=33098 max_rank=4 min=31427 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=37747 max_load_reduction_pct=0.532811 moved_to_replicas=149380 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[32307, 33096, 33097, 33095, 33096, 33096, 32147, 32210] avg=32768.0000 max=33097 max_rank=2 min=32147 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25123 max_load_reduction_pct=0.431518 moved_to_replicas=94647 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[24134, 33589, 9478, 58220, 53756, 18610, 32147, 32210] avg=32768.0000 max=58220 max_rank=3 min=9478 min_rank=2 imbalance_ratio=1.776733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[30491, 65524, 1392, 28822, 4467, 65592, 3795, 62061] avg=32768.0000 max=65592 max_rank=5 min=1392 min_rank=2 imbalance_ratio=2.001709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[32307, 33096, 33097, 33095, 33096, 33096, 32147, 32210] avg=32768.0000 max=33097 max_rank=2 min=32147 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25123 max_load_reduction_pct=0.431518 moved_to_replicas=94647 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[32307, 33096, 33097, 33095, 33096, 33096, 32147, 32210] avg=32768.0000 max=33097 max_rank=2 min=32147 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25123 max_load_reduction_pct=0.431518 moved_to_replicas=94647 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[24134, 33589, 9478, 58220, 53756, 18610, 32147, 32210] avg=32768.0000 max=58220 max_rank=3 min=9478 min_rank=2 imbalance_ratio=1.776733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[32307, 33096, 33097, 33095, 33096, 33096, 32147, 32210] avg=32768.0000 max=33097 max_rank=2 min=32147 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25123 max_load_reduction_pct=0.431518 moved_to_replicas=94647 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[15126, 8728, 3159, 28139, 58291, 79998, 2513, 66190] avg=32768.0000 max=79998 max_rank=5 min=2513 min_rank=6 imbalance_ratio=2.441345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[24134, 33589, 9478, 58220, 53756, 18610, 32147, 32210] avg=32768.0000 max=58220 max_rank=3 min=9478 min_rank=2 imbalance_ratio=1.776733 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[30674, 33096, 33096, 32894, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30674 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32496 max_load_reduction_pct=0.495426 moved_to_replicas=192459 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[30491, 65524, 1392, 28822, 4467, 65592, 3795, 62061] avg=32768.0000 max=65592 max_rank=5 min=1392 min_rank=2 imbalance_ratio=2.001709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[32307, 33096, 33097, 33095, 33096, 33096, 32147, 32210] avg=32768.0000 max=33097 max_rank=2 min=32147 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25123 max_load_reduction_pct=0.431518 moved_to_replicas=94647 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[30491, 65524, 1392, 28822, 4467, 65592, 3795, 62061] avg=32768.0000 max=65592 max_rank=5 min=1392 min_rank=2 imbalance_ratio=2.001709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[33097, 33096, 33095, 30472, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46901 max_load_reduction_pct=0.586277 moved_to_replicas=221275 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[32307, 33096, 33097, 33095, 33096, 33096, 32147, 32210] avg=32768.0000 max=33097 max_rank=2 min=32147 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=25123 max_load_reduction_pct=0.431518 moved_to_replicas=94647 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[30491, 65524, 1392, 28822, 4467, 65592, 3795, 62061] avg=32768.0000 max=65592 max_rank=5 min=1392 min_rank=2 imbalance_ratio=2.001709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[30491, 65524, 1392, 28822, 4467, 65592, 3795, 62061] avg=32768.0000 max=65592 max_rank=5 min=1392 min_rank=2 imbalance_ratio=2.001709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[30674, 33096, 33096, 32894, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30674 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32496 max_load_reduction_pct=0.495426 moved_to_replicas=192459 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[15126, 8728, 3159, 28139, 58291, 79998, 2513, 66190] avg=32768.0000 max=79998 max_rank=5 min=2513 min_rank=6 imbalance_ratio=2.441345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[30674, 33096, 33096, 32894, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30674 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32496 max_load_reduction_pct=0.495426 moved_to_replicas=192459 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[30674, 33096, 33096, 32894, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30674 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32496 max_load_reduction_pct=0.495426 moved_to_replicas=192459 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[30491, 65524, 1392, 28822, 4467, 65592, 3795, 62061] avg=32768.0000 max=65592 max_rank=5 min=1392 min_rank=2 imbalance_ratio=2.001709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[30674, 33096, 33096, 32894, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30674 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32496 max_load_reduction_pct=0.495426 moved_to_replicas=192459 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[64300, 56564, 32184, 11614, 59, 48891, 15670, 32862] avg=32768.0000 max=64300 max_rank=0 min=59 min_rank=4 imbalance_ratio=1.962280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[30491, 65524, 1392, 28822, 4467, 65592, 3795, 62061] avg=32768.0000 max=65592 max_rank=5 min=1392 min_rank=2 imbalance_ratio=2.001709 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[33097, 33096, 33095, 30472, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46901 max_load_reduction_pct=0.586277 moved_to_replicas=221275 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[15126, 8728, 3159, 28139, 58291, 79998, 2513, 66190] avg=32768.0000 max=79998 max_rank=5 min=2513 min_rank=6 imbalance_ratio=2.441345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[30674, 33096, 33096, 32894, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30674 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32496 max_load_reduction_pct=0.495426 moved_to_replicas=192459 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[33096, 33095, 32184, 33097, 31618, 33097, 33095, 32862] avg=32768.0000 max=33097 max_rank=3 min=31618 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31203 max_load_reduction_pct=0.485272 moved_to_replicas=151824 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[30674, 33096, 33096, 32894, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=30674 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32496 max_load_reduction_pct=0.495426 moved_to_replicas=192459 active_clone_links=16 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[15126, 8728, 3159, 28139, 58291, 79998, 2513, 66190] avg=32768.0000 max=79998 max_rank=5 min=2513 min_rank=6 imbalance_ratio=2.441345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[15126, 8728, 3159, 28139, 58291, 79998, 2513, 66190] avg=32768.0000 max=79998 max_rank=5 min=2513 min_rank=6 imbalance_ratio=2.441345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[64300, 56564, 32184, 11614, 59, 48891, 15670, 32862] avg=32768.0000 max=64300 max_rank=0 min=59 min_rank=4 imbalance_ratio=1.962280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[15126, 8728, 3159, 28139, 58291, 79998, 2513, 66190] avg=32768.0000 max=79998 max_rank=5 min=2513 min_rank=6 imbalance_ratio=2.441345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[33097, 33096, 33095, 30472, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46901 max_load_reduction_pct=0.586277 moved_to_replicas=221275 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[33097, 33096, 33095, 30472, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46901 max_load_reduction_pct=0.586277 moved_to_replicas=221275 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[33097, 33096, 33095, 30472, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46901 max_load_reduction_pct=0.586277 moved_to_replicas=221275 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[15126, 8728, 3159, 28139, 58291, 79998, 2513, 66190] avg=32768.0000 max=79998 max_rank=5 min=2513 min_rank=6 imbalance_ratio=2.441345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[34335, 5150, 1097, 84037, 37187, 32161, 15752, 52425] avg=32768.0000 max=84037 max_rank=3 min=1097 min_rank=2 imbalance_ratio=2.564606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[15126, 8728, 3159, 28139, 58291, 79998, 2513, 66190] avg=32768.0000 max=79998 max_rank=5 min=2513 min_rank=6 imbalance_ratio=2.441345 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[33096, 33095, 32184, 33097, 31618, 33097, 33095, 32862] avg=32768.0000 max=33097 max_rank=3 min=31618 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31203 max_load_reduction_pct=0.485272 moved_to_replicas=151824 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[33097, 33096, 33095, 30472, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46901 max_load_reduction_pct=0.586277 moved_to_replicas=221275 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[33097, 33096, 33095, 30472, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46901 max_load_reduction_pct=0.586277 moved_to_replicas=221275 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[33096, 31407, 33096, 33096, 33096, 32161, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31407 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=50940 max_load_reduction_pct=0.606162 moved_to_replicas=155581 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[33097, 33096, 33095, 30472, 33095, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=46901 max_load_reduction_pct=0.586277 moved_to_replicas=221275 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[64300, 56564, 32184, 11614, 59, 48891, 15670, 32862] avg=32768.0000 max=64300 max_rank=0 min=59 min_rank=4 imbalance_ratio=1.962280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[64300, 56564, 32184, 11614, 59, 48891, 15670, 32862] avg=32768.0000 max=64300 max_rank=0 min=59 min_rank=4 imbalance_ratio=1.962280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[34335, 5150, 1097, 84037, 37187, 32161, 15752, 52425] avg=32768.0000 max=84037 max_rank=3 min=1097 min_rank=2 imbalance_ratio=2.564606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[64300, 56564, 32184, 11614, 59, 48891, 15670, 32862] avg=32768.0000 max=64300 max_rank=0 min=59 min_rank=4 imbalance_ratio=1.962280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[64300, 56564, 32184, 11614, 59, 48891, 15670, 32862] avg=32768.0000 max=64300 max_rank=0 min=59 min_rank=4 imbalance_ratio=1.962280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[33096, 33095, 32184, 33097, 31618, 33097, 33095, 32862] avg=32768.0000 max=33097 max_rank=3 min=31618 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31203 max_load_reduction_pct=0.485272 moved_to_replicas=151824 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[33096, 33095, 32184, 33097, 31618, 33097, 33095, 32862] avg=32768.0000 max=33097 max_rank=3 min=31618 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31203 max_load_reduction_pct=0.485272 moved_to_replicas=151824 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[31616, 63123, 16474, 29752, 60609, 1332, 20913, 38325] avg=32768.0000 max=63123 max_rank=1 min=1332 min_rank=5 imbalance_ratio=1.926361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[64300, 56564, 32184, 11614, 59, 48891, 15670, 32862] avg=32768.0000 max=64300 max_rank=0 min=59 min_rank=4 imbalance_ratio=1.962280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[64300, 56564, 32184, 11614, 59, 48891, 15670, 32862] avg=32768.0000 max=64300 max_rank=0 min=59 min_rank=4 imbalance_ratio=1.962280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[33096, 31407, 33096, 33096, 33096, 32161, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31407 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=50940 max_load_reduction_pct=0.606162 moved_to_replicas=155581 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[33096, 33095, 32184, 33097, 31618, 33097, 33095, 32862] avg=32768.0000 max=33097 max_rank=3 min=31618 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31203 max_load_reduction_pct=0.485272 moved_to_replicas=151824 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[33096, 33095, 32184, 33097, 31618, 33097, 33095, 32862] avg=32768.0000 max=33097 max_rank=3 min=31618 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31203 max_load_reduction_pct=0.485272 moved_to_replicas=151824 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[31616, 33096, 33096, 33096, 33096, 32880, 32168, 33096] avg=32768.0000 max=33096 max_rank=1 min=31616 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30027 max_load_reduction_pct=0.475690 moved_to_replicas=128167 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[33096, 33095, 32184, 33097, 31618, 33097, 33095, 32862] avg=32768.0000 max=33097 max_rank=3 min=31618 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31203 max_load_reduction_pct=0.485272 moved_to_replicas=151824 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[33096, 33095, 32184, 33097, 31618, 33097, 33095, 32862] avg=32768.0000 max=33097 max_rank=3 min=31618 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=31203 max_load_reduction_pct=0.485272 moved_to_replicas=151824 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[34335, 5150, 1097, 84037, 37187, 32161, 15752, 52425] avg=32768.0000 max=84037 max_rank=3 min=1097 min_rank=2 imbalance_ratio=2.564606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[34335, 5150, 1097, 84037, 37187, 32161, 15752, 52425] avg=32768.0000 max=84037 max_rank=3 min=1097 min_rank=2 imbalance_ratio=2.564606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[31616, 63123, 16474, 29752, 60609, 1332, 20913, 38325] avg=32768.0000 max=63123 max_rank=1 min=1332 min_rank=5 imbalance_ratio=1.926361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[34335, 5150, 1097, 84037, 37187, 32161, 15752, 52425] avg=32768.0000 max=84037 max_rank=3 min=1097 min_rank=2 imbalance_ratio=2.564606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[34335, 5150, 1097, 84037, 37187, 32161, 15752, 52425] avg=32768.0000 max=84037 max_rank=3 min=1097 min_rank=2 imbalance_ratio=2.564606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[34945, 50934, 38130, 10231, 20805, 20304, 40756, 46039] avg=32768.0000 max=50934 max_rank=1 min=10231 min_rank=3 imbalance_ratio=1.554382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[33096, 31407, 33096, 33096, 33096, 32161, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31407 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=50940 max_load_reduction_pct=0.606162 moved_to_replicas=155581 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[33096, 31407, 33096, 33096, 33096, 32161, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31407 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=50940 max_load_reduction_pct=0.606162 moved_to_replicas=155581 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[34335, 5150, 1097, 84037, 37187, 32161, 15752, 52425] avg=32768.0000 max=84037 max_rank=3 min=1097 min_rank=2 imbalance_ratio=2.564606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[34335, 5150, 1097, 84037, 37187, 32161, 15752, 52425] avg=32768.0000 max=84037 max_rank=3 min=1097 min_rank=2 imbalance_ratio=2.564606 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[33096, 31407, 33096, 33096, 33096, 32161, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31407 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=50940 max_load_reduction_pct=0.606162 moved_to_replicas=155581 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[33096, 31407, 33096, 33096, 33096, 32161, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31407 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=50940 max_load_reduction_pct=0.606162 moved_to_replicas=155581 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[31616, 33096, 33096, 33096, 33096, 32880, 32168, 33096] avg=32768.0000 max=33096 max_rank=1 min=31616 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30027 max_load_reduction_pct=0.475690 moved_to_replicas=128167 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33097, 33097, 33094, 30472, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=17837 max_load_reduction_pct=0.350198 moved_to_replicas=104211 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[33096, 31407, 33096, 33096, 33096, 32161, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31407 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=50940 max_load_reduction_pct=0.606162 moved_to_replicas=155581 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[33096, 31407, 33096, 33096, 33096, 32161, 33097, 33095] avg=32768.0000 max=33097 max_rank=6 min=31407 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=50940 max_load_reduction_pct=0.606162 moved_to_replicas=155581 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[31616, 63123, 16474, 29752, 60609, 1332, 20913, 38325] avg=32768.0000 max=63123 max_rank=1 min=1332 min_rank=5 imbalance_ratio=1.926361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[31616, 63123, 16474, 29752, 60609, 1332, 20913, 38325] avg=32768.0000 max=63123 max_rank=1 min=1332 min_rank=5 imbalance_ratio=1.926361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[31616, 63123, 16474, 29752, 60609, 1332, 20913, 38325] avg=32768.0000 max=63123 max_rank=1 min=1332 min_rank=5 imbalance_ratio=1.926361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[31616, 63123, 16474, 29752, 60609, 1332, 20913, 38325] avg=32768.0000 max=63123 max_rank=1 min=1332 min_rank=5 imbalance_ratio=1.926361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[34945, 50934, 38130, 10231, 20805, 20304, 40756, 46039] avg=32768.0000 max=50934 max_rank=1 min=10231 min_rank=3 imbalance_ratio=1.554382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[347, 1740, 43462, 9115, 97555, 64788, 9308, 35829] avg=32768.0000 max=97555 max_rank=4 min=347 min_rank=0 imbalance_ratio=2.977142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[31616, 33096, 33096, 33096, 33096, 32880, 32168, 33096] avg=32768.0000 max=33096 max_rank=1 min=31616 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30027 max_load_reduction_pct=0.475690 moved_to_replicas=128167 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[31616, 63123, 16474, 29752, 60609, 1332, 20913, 38325] avg=32768.0000 max=63123 max_rank=1 min=1332 min_rank=5 imbalance_ratio=1.926361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[31616, 63123, 16474, 29752, 60609, 1332, 20913, 38325] avg=32768.0000 max=63123 max_rank=1 min=1332 min_rank=5 imbalance_ratio=1.926361 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[31616, 33096, 33096, 33096, 33096, 32880, 32168, 33096] avg=32768.0000 max=33096 max_rank=1 min=31616 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30027 max_load_reduction_pct=0.475690 moved_to_replicas=128167 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[31616, 33096, 33096, 33096, 33096, 32880, 32168, 33096] avg=32768.0000 max=33096 max_rank=1 min=31616 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30027 max_load_reduction_pct=0.475690 moved_to_replicas=128167 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[31616, 33096, 33096, 33096, 33096, 32880, 32168, 33096] avg=32768.0000 max=33096 max_rank=1 min=31616 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30027 max_load_reduction_pct=0.475690 moved_to_replicas=128167 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33097, 33097, 33094, 30472, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=17837 max_load_reduction_pct=0.350198 moved_to_replicas=104211 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[32916, 33096, 33096, 30653, 33096, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=30653 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64459 max_load_reduction_pct=0.660745 moved_to_replicas=228360 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[31616, 33096, 33096, 33096, 33096, 32880, 32168, 33096] avg=32768.0000 max=33096 max_rank=1 min=31616 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30027 max_load_reduction_pct=0.475690 moved_to_replicas=128167 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[31616, 33096, 33096, 33096, 33096, 32880, 32168, 33096] avg=32768.0000 max=33096 max_rank=1 min=31616 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=30027 max_load_reduction_pct=0.475690 moved_to_replicas=128167 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[34945, 50934, 38130, 10231, 20805, 20304, 40756, 46039] avg=32768.0000 max=50934 max_rank=1 min=10231 min_rank=3 imbalance_ratio=1.554382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[34945, 50934, 38130, 10231, 20805, 20304, 40756, 46039] avg=32768.0000 max=50934 max_rank=1 min=10231 min_rank=3 imbalance_ratio=1.554382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[34945, 50934, 38130, 10231, 20805, 20304, 40756, 46039] avg=32768.0000 max=50934 max_rank=1 min=10231 min_rank=3 imbalance_ratio=1.554382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[34945, 50934, 38130, 10231, 20805, 20304, 40756, 46039] avg=32768.0000 max=50934 max_rank=1 min=10231 min_rank=3 imbalance_ratio=1.554382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[347, 1740, 43462, 9115, 97555, 64788, 9308, 35829] avg=32768.0000 max=97555 max_rank=4 min=347 min_rank=0 imbalance_ratio=2.977142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[82131, 52558, 3345, 746, 9435, 41167, 32756, 40006] avg=32768.0000 max=82131 max_rank=0 min=746 min_rank=3 imbalance_ratio=2.506439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33097, 33097, 33094, 30472, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=17837 max_load_reduction_pct=0.350198 moved_to_replicas=104211 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[34945, 50934, 38130, 10231, 20805, 20304, 40756, 46039] avg=32768.0000 max=50934 max_rank=1 min=10231 min_rank=3 imbalance_ratio=1.554382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[34945, 50934, 38130, 10231, 20805, 20304, 40756, 46039] avg=32768.0000 max=50934 max_rank=1 min=10231 min_rank=3 imbalance_ratio=1.554382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33097, 33097, 33094, 30472, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=17837 max_load_reduction_pct=0.350198 moved_to_replicas=104211 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33097, 33097, 33094, 30472, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=17837 max_load_reduction_pct=0.350198 moved_to_replicas=104211 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33097, 33097, 33094, 30472, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=17837 max_load_reduction_pct=0.350198 moved_to_replicas=104211 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[32916, 33096, 33096, 30653, 33096, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=30653 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64459 max_load_reduction_pct=0.660745 moved_to_replicas=228360 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 30812, 33096, 32756, 33097] avg=32768.0000 max=33097 max_rank=7 min=30812 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49034 max_load_reduction_pct=0.597022 moved_to_replicas=167469 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33097, 33097, 33094, 30472, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=17837 max_load_reduction_pct=0.350198 moved_to_replicas=104211 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[33096, 33097, 33097, 33094, 30472, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=17837 max_load_reduction_pct=0.350198 moved_to_replicas=104211 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[347, 1740, 43462, 9115, 97555, 64788, 9308, 35829] avg=32768.0000 max=97555 max_rank=4 min=347 min_rank=0 imbalance_ratio=2.977142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[347, 1740, 43462, 9115, 97555, 64788, 9308, 35829] avg=32768.0000 max=97555 max_rank=4 min=347 min_rank=0 imbalance_ratio=2.977142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[347, 1740, 43462, 9115, 97555, 64788, 9308, 35829] avg=32768.0000 max=97555 max_rank=4 min=347 min_rank=0 imbalance_ratio=2.977142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[347, 1740, 43462, 9115, 97555, 64788, 9308, 35829] avg=32768.0000 max=97555 max_rank=4 min=347 min_rank=0 imbalance_ratio=2.977142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[82131, 52558, 3345, 746, 9435, 41167, 32756, 40006] avg=32768.0000 max=82131 max_rank=0 min=746 min_rank=3 imbalance_ratio=2.506439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[39837, 57024, 30527, 1317, 32729, 10217, 31308, 59185] avg=32768.0000 max=59185 max_rank=7 min=1317 min_rank=3 imbalance_ratio=1.806183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[347, 1740, 43462, 9115, 97555, 64788, 9308, 35829] avg=32768.0000 max=97555 max_rank=4 min=347 min_rank=0 imbalance_ratio=2.977142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[32916, 33096, 33096, 30653, 33096, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=30653 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64459 max_load_reduction_pct=0.660745 moved_to_replicas=228360 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[347, 1740, 43462, 9115, 97555, 64788, 9308, 35829] avg=32768.0000 max=97555 max_rank=4 min=347 min_rank=0 imbalance_ratio=2.977142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[32916, 33096, 33096, 30653, 33096, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=30653 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64459 max_load_reduction_pct=0.660745 moved_to_replicas=228360 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[32916, 33096, 33096, 30653, 33096, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=30653 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64459 max_load_reduction_pct=0.660745 moved_to_replicas=228360 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[32916, 33096, 33096, 30653, 33096, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=30653 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64459 max_load_reduction_pct=0.660745 moved_to_replicas=228360 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 30812, 33096, 32756, 33097] avg=32768.0000 max=33097 max_rank=7 min=30812 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49034 max_load_reduction_pct=0.597022 moved_to_replicas=167469 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[33095, 33096, 32627, 33096, 32729, 33096, 31308, 33097] avg=32768.0000 max=33097 max_rank=7 min=31308 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26088 max_load_reduction_pct=0.440787 moved_to_replicas=116576 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[32916, 33096, 33096, 30653, 33096, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=30653 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64459 max_load_reduction_pct=0.660745 moved_to_replicas=228360 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[32916, 33096, 33096, 30653, 33096, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=30653 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=64459 max_load_reduction_pct=0.660745 moved_to_replicas=228360 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[82131, 52558, 3345, 746, 9435, 41167, 32756, 40006] avg=32768.0000 max=82131 max_rank=0 min=746 min_rank=3 imbalance_ratio=2.506439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[82131, 52558, 3345, 746, 9435, 41167, 32756, 40006] avg=32768.0000 max=82131 max_rank=0 min=746 min_rank=3 imbalance_ratio=2.506439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[82131, 52558, 3345, 746, 9435, 41167, 32756, 40006] avg=32768.0000 max=82131 max_rank=0 min=746 min_rank=3 imbalance_ratio=2.506439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[82131, 52558, 3345, 746, 9435, 41167, 32756, 40006] avg=32768.0000 max=82131 max_rank=0 min=746 min_rank=3 imbalance_ratio=2.506439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[39837, 57024, 30527, 1317, 32729, 10217, 31308, 59185] avg=32768.0000 max=59185 max_rank=7 min=1317 min_rank=3 imbalance_ratio=1.806183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[82131, 52558, 3345, 746, 9435, 41167, 32756, 40006] avg=32768.0000 max=82131 max_rank=0 min=746 min_rank=3 imbalance_ratio=2.506439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 30812, 33096, 32756, 33097] avg=32768.0000 max=33097 max_rank=7 min=30812 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49034 max_load_reduction_pct=0.597022 moved_to_replicas=167469 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[82131, 52558, 3345, 746, 9435, 41167, 32756, 40006] avg=32768.0000 max=82131 max_rank=0 min=746 min_rank=3 imbalance_ratio=2.506439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 30812, 33096, 32756, 33097] avg=32768.0000 max=33097 max_rank=7 min=30812 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49034 max_load_reduction_pct=0.597022 moved_to_replicas=167469 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 30812, 33096, 32756, 33097] avg=32768.0000 max=33097 max_rank=7 min=30812 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49034 max_load_reduction_pct=0.597022 moved_to_replicas=167469 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 30812, 33096, 32756, 33097] avg=32768.0000 max=33097 max_rank=7 min=30812 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49034 max_load_reduction_pct=0.597022 moved_to_replicas=167469 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[33095, 33096, 32627, 33096, 32729, 33096, 31308, 33097] avg=32768.0000 max=33097 max_rank=7 min=31308 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26088 max_load_reduction_pct=0.440787 moved_to_replicas=116576 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 30812, 33096, 32756, 33097] avg=32768.0000 max=33097 max_rank=7 min=30812 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49034 max_load_reduction_pct=0.597022 moved_to_replicas=167469 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[33095, 33096, 33096, 33096, 30812, 33096, 32756, 33097] avg=32768.0000 max=33097 max_rank=7 min=30812 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=49034 max_load_reduction_pct=0.597022 moved_to_replicas=167469 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[39837, 57024, 30527, 1317, 32729, 10217, 31308, 59185] avg=32768.0000 max=59185 max_rank=7 min=1317 min_rank=3 imbalance_ratio=1.806183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[39837, 57024, 30527, 1317, 32729, 10217, 31308, 59185] avg=32768.0000 max=59185 max_rank=7 min=1317 min_rank=3 imbalance_ratio=1.806183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[39837, 57024, 30527, 1317, 32729, 10217, 31308, 59185] avg=32768.0000 max=59185 max_rank=7 min=1317 min_rank=3 imbalance_ratio=1.806183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[39837, 57024, 30527, 1317, 32729, 10217, 31308, 59185] avg=32768.0000 max=59185 max_rank=7 min=1317 min_rank=3 imbalance_ratio=1.806183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[39837, 57024, 30527, 1317, 32729, 10217, 31308, 59185] avg=32768.0000 max=59185 max_rank=7 min=1317 min_rank=3 imbalance_ratio=1.806183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[39837, 57024, 30527, 1317, 32729, 10217, 31308, 59185] avg=32768.0000 max=59185 max_rank=7 min=1317 min_rank=3 imbalance_ratio=1.806183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[33095, 33096, 32627, 33096, 32729, 33096, 31308, 33097] avg=32768.0000 max=33097 max_rank=7 min=31308 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26088 max_load_reduction_pct=0.440787 moved_to_replicas=116576 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[33095, 33096, 32627, 33096, 32729, 33096, 31308, 33097] avg=32768.0000 max=33097 max_rank=7 min=31308 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26088 max_load_reduction_pct=0.440787 moved_to_replicas=116576 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[33095, 33096, 32627, 33096, 32729, 33096, 31308, 33097] avg=32768.0000 max=33097 max_rank=7 min=31308 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26088 max_load_reduction_pct=0.440787 moved_to_replicas=116576 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[33095, 33096, 32627, 33096, 32729, 33096, 31308, 33097] avg=32768.0000 max=33097 max_rank=7 min=31308 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26088 max_load_reduction_pct=0.440787 moved_to_replicas=116576 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[33095, 33096, 32627, 33096, 32729, 33096, 31308, 33097] avg=32768.0000 max=33097 max_rank=7 min=31308 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26088 max_load_reduction_pct=0.440787 moved_to_replicas=116576 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[33095, 33096, 32627, 33096, 32729, 33096, 31308, 33097] avg=32768.0000 max=33097 max_rank=7 min=31308 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=26088 max_load_reduction_pct=0.440787 moved_to_replicas=116576 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=32 total_expected_recv=1053861 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=246912 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=32 total_expected_recv=1053393 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=245760 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=32 total_expected_recv=1049921 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=251392 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=32 total_expected_recv=1057220 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=244096 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=32 total_expected_recv=1046207 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253056 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=32 total_expected_recv=1041961 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=257536 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=32 total_expected_recv=1041822 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=257280 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=32 total_expected_recv=1044223 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254336 peak_demand=33097 required_capacity=1.010040 + [2026-08-30 05:10:27] iteration 3/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 576.2 | throughput per GPU (TFLOP/s/GPU): 578.6 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.028937E+01 | loss scale: 1.0 | grad norm: 201.646 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=40960 expected_recv=64827 expected_overflow=0 exact_recv_per_rank=[33095, 31731, 33097, 31837, 33096, 33095, 33098, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 31731, 33097, 31837, 33096, 33095, 33098, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 31731, 33097, 31837, 33096, 33095, 33098, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=40960 expected_recv=64933 expected_overflow=0 exact_recv_per_rank=[33095, 31731, 33097, 31837, 33096, 33095, 33098, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33095, 31731, 33097, 31837, 33096, 33095, 33098, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33095, 31731, 33097, 31837, 33096, 33095, 33098, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=40960 expected_recv=63567 expected_overflow=0 exact_recv_per_rank=[33095, 31731, 33097, 31837, 33096, 33095, 33098, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=2 budget=40960 expected_recv=64827 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 31731, 33097, 31837, 33096, 33095, 33098, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=2 budget=40960 expected_recv=64933 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=2 budget=40960 expected_recv=63567 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32059, 32250, 32778, 32673, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65638 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32059, 32250, 32778, 32673, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32059, 32250, 32778, 32673, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=40960 expected_recv=64243 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32059, 32250, 32778, 32673, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65346 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32059, 32250, 32778, 32673, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32059, 32250, 32778, 32673, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65638 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=40960 expected_recv=64927 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32059, 32250, 32778, 32673, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=2 budget=40960 expected_recv=64243 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65346 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=2 budget=40960 expected_recv=64927 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32059, 32250, 32778, 32673, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=2 budget=40960 expected_recv=65558 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32839, 33096, 31891, 33096, 33096, 33046, 32286, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=40960 expected_recv=65726 expected_overflow=0 exact_recv_per_rank=[32839, 33096, 31891, 33096, 33096, 33046, 32286, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=40960 expected_recv=66081 expected_overflow=0 exact_recv_per_rank=[32839, 33096, 31891, 33096, 33096, 33046, 32286, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=40960 expected_recv=64987 expected_overflow=0 exact_recv_per_rank=[32839, 33096, 31891, 33096, 33096, 33046, 32286, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32839, 33096, 31891, 33096, 33096, 33046, 32286, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=2 budget=40960 expected_recv=65726 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=2 budget=40960 expected_recv=66081 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=40960 expected_recv=64154 expected_overflow=0 exact_recv_per_rank=[32839, 33096, 31891, 33096, 33096, 33046, 32286, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=40960 expected_recv=64764 expected_overflow=0 exact_recv_per_rank=[32839, 33096, 31891, 33096, 33096, 33046, 32286, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=2 budget=40960 expected_recv=64987 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=2 budget=40960 expected_recv=64154 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=2 budget=40960 expected_recv=64764 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32839, 33096, 31891, 33096, 33096, 33046, 32286, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=40960 expected_recv=63916 expected_overflow=0 exact_recv_per_rank=[33096, 31065, 32881, 33098, 33096, 33096, 33034, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31065, 32881, 33098, 33096, 33096, 33034, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31065, 32881, 33098, 33096, 33096, 33034, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=2 budget=40960 expected_recv=63916 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65736 expected_overflow=0 exact_recv_per_rank=[33096, 31065, 32881, 33098, 33096, 33096, 33034, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33096, 31065, 32881, 33098, 33096, 33096, 33034, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65874 expected_overflow=0 exact_recv_per_rank=[33096, 31065, 32881, 33098, 33096, 33096, 33034, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65736 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65272 expected_overflow=0 exact_recv_per_rank=[33096, 31065, 32881, 33098, 33096, 33096, 33034, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65874 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=40960 expected_recv=64912 expected_overflow=0 exact_recv_per_rank=[33096, 31065, 32881, 33098, 33096, 33096, 33034, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=2 budget=40960 expected_recv=65272 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=2 budget=40960 expected_recv=64912 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65209 expected_overflow=0 exact_recv_per_rank=[31459, 33095, 33096, 32790, 33096, 32759, 32753, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[31459, 33095, 33096, 32790, 33096, 32759, 32753, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=40960 expected_recv=63752 expected_overflow=0 exact_recv_per_rank=[31459, 33095, 33096, 32790, 33096, 32759, 32753, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65209 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=2 budget=40960 expected_recv=63752 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31459, 33095, 33096, 32790, 33096, 32759, 32753, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65886 expected_overflow=0 exact_recv_per_rank=[31459, 33095, 33096, 32790, 33096, 32759, 32753, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31459, 33095, 33096, 32790, 33096, 32759, 32753, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65454 expected_overflow=0 exact_recv_per_rank=[31459, 33095, 33096, 32790, 33096, 32759, 32753, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65886 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65411 expected_overflow=0 exact_recv_per_rank=[31459, 33095, 33096, 32790, 33096, 32759, 32753, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65454 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=2 budget=40960 expected_recv=65411 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32794, 32718, 32111, 32374, 32859, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65355 expected_overflow=0 exact_recv_per_rank=[32794, 32718, 32111, 32374, 32859, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65684 expected_overflow=0 exact_recv_per_rank=[32794, 32718, 32111, 32374, 32859, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=40960 expected_recv=64613 expected_overflow=0 exact_recv_per_rank=[32794, 32718, 32111, 32374, 32859, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65355 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65684 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=40960 expected_recv=64789 expected_overflow=0 exact_recv_per_rank=[32794, 32718, 32111, 32374, 32859, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32794, 32718, 32111, 32374, 32859, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32794, 32718, 32111, 32374, 32859, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=2 budget=40960 expected_recv=64613 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65271 expected_overflow=0 exact_recv_per_rank=[32794, 32718, 32111, 32374, 32859, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=2 budget=40960 expected_recv=64789 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=2 budget=40960 expected_recv=65271 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=40960 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[33095, 32967, 32772, 32804, 33098, 33094, 31534, 32780] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=40960 expected_recv=66063 expected_overflow=0 exact_recv_per_rank=[33095, 32967, 32772, 32804, 33098, 33094, 31534, 32780] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=2 budget=40960 expected_recv=66190 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 32967, 32772, 32804, 33098, 33094, 31534, 32780] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=40960 expected_recv=65868 expected_overflow=0 exact_recv_per_rank=[33095, 32967, 32772, 32804, 33098, 33094, 31534, 32780] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=2 budget=40960 expected_recv=66063 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=2 budget=40960 expected_recv=65868 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33095, 32967, 32772, 32804, 33098, 33094, 31534, 32780] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=40960 expected_recv=65584 expected_overflow=0 exact_recv_per_rank=[33095, 32967, 32772, 32804, 33098, 33094, 31534, 32780] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=40960 expected_recv=65604 expected_overflow=0 exact_recv_per_rank=[33095, 32967, 32772, 32804, 33098, 33094, 31534, 32780] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=40960 expected_recv=62594 expected_overflow=0 exact_recv_per_rank=[33095, 32967, 32772, 32804, 33098, 33094, 31534, 32780] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=2 budget=40960 expected_recv=65584 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=2 budget=40960 expected_recv=65604 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=2 budget=40960 expected_recv=62594 actual_valid_recv=63616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=40960 expected_recv=64363 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 32463, 33096, 31105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=63616 slack_slots=18304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=2 budget=40960 expected_recv=64363 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 32463, 33096, 31105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 32463, 33096, 31105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 32463, 33096, 31105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 32463, 33096, 31105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=40960 expected_recv=63000 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 32463, 33096, 31105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=40960 expected_recv=65964 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 32463, 33096, 31105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 32463, 33096, 31105] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=40960 expected_recv=63960 expected_overflow=0 exact_recv_per_rank=[33096, 32781, 33096, 32786, 33096, 31903, 33095, 32291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=2 budget=40960 expected_recv=63000 actual_valid_recv=64000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=2 budget=40960 expected_recv=65964 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=2 budget=40960 expected_recv=63960 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=64000 slack_slots=17920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65512 expected_overflow=0 exact_recv_per_rank=[33096, 32781, 33096, 32786, 33096, 31903, 33095, 32291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32781, 33096, 32786, 33096, 31903, 33095, 32291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65512 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65798 expected_overflow=0 exact_recv_per_rank=[33096, 32781, 33096, 32786, 33096, 31903, 33095, 32291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65798 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32781, 33096, 32786, 33096, 31903, 33095, 32291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=40960 expected_recv=64958 expected_overflow=0 exact_recv_per_rank=[33096, 32781, 33096, 32786, 33096, 31903, 33095, 32291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65589 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32560, 32658, 32976, 32755, 32977, 32026] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65484 expected_overflow=0 exact_recv_per_rank=[33096, 32781, 33096, 32786, 33096, 31903, 33095, 32291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32781, 33096, 32786, 33096, 31903, 33095, 32291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=2 budget=40960 expected_recv=64958 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65589 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=2 budget=40960 expected_recv=65484 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32560, 32658, 32976, 32755, 32977, 32026] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32560, 32658, 32976, 32755, 32977, 32026] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65421 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32560, 32658, 32976, 32755, 32977, 32026] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65841 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32560, 32658, 32976, 32755, 32977, 32026] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65556 expected_overflow=0 exact_recv_per_rank=[32450, 32710, 33096, 32840, 32054, 32801, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65421 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=40960 expected_recv=63410 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32560, 32658, 32976, 32755, 32977, 32026] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65570 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32560, 32658, 32976, 32755, 32977, 32026] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65841 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=40960 expected_recv=66073 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32560, 32658, 32976, 32755, 32977, 32026] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65556 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=2 budget=40960 expected_recv=63410 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=2 budget=40960 expected_recv=65570 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[32450, 32710, 33096, 32840, 32054, 32801, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=2 budget=40960 expected_recv=66073 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=40960 expected_recv=64587 expected_overflow=0 exact_recv_per_rank=[32450, 32710, 33096, 32840, 32054, 32801, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65523 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=2 budget=40960 expected_recv=64587 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32450, 32710, 33096, 32840, 32054, 32801, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=40960 expected_recv=64109 expected_overflow=0 exact_recv_per_rank=[32450, 32710, 33096, 32840, 32054, 32801, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=40960 expected_recv=65378 expected_overflow=0 exact_recv_per_rank=[32252, 32833, 33096, 31944, 33096, 32730, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32450, 32710, 33096, 32840, 32054, 32801, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=2 budget=40960 expected_recv=64109 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65936 expected_overflow=0 exact_recv_per_rank=[32450, 32710, 33096, 32840, 32054, 32801, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=2 budget=40960 expected_recv=65378 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32450, 32710, 33096, 32840, 32054, 32801, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=40960 expected_recv=65684 expected_overflow=0 exact_recv_per_rank=[32252, 32833, 33096, 31944, 33096, 32730, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=40960 expected_recv=64979 expected_overflow=0 exact_recv_per_rank=[32252, 32833, 33096, 31944, 33096, 32730, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=2 budget=40960 expected_recv=65936 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=2 budget=40960 expected_recv=65684 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=2 budget=40960 expected_recv=64979 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32252, 32833, 33096, 31944, 33096, 32730, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32252, 32833, 33096, 31944, 33096, 32730, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=40960 expected_recv=63816 expected_overflow=0 exact_recv_per_rank=[33096, 32913, 32636, 32775, 33096, 32510, 32273, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32252, 32833, 33096, 31944, 33096, 32730, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32252, 32833, 33096, 31944, 33096, 32730, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=40960 expected_recv=63478 expected_overflow=0 exact_recv_per_rank=[32252, 32833, 33096, 31944, 33096, 32730, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=12 calls=2 budget=40960 expected_recv=63816 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65693 expected_overflow=0 exact_recv_per_rank=[33096, 32913, 32636, 32775, 33096, 32510, 32273, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32913, 32636, 32775, 33096, 32510, 32273, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=2 budget=40960 expected_recv=63478 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65693 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65214 expected_overflow=0 exact_recv_per_rank=[33096, 32913, 32636, 32775, 33096, 32510, 32273, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32913, 32636, 32775, 33096, 32510, 32273, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65214 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65862 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 33097, 31895, 32766, 32355, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65941 expected_overflow=0 exact_recv_per_rank=[33096, 32913, 32636, 32775, 33096, 32510, 32273, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65369 expected_overflow=0 exact_recv_per_rank=[33096, 32913, 32636, 32775, 33096, 32510, 32273, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65862 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 33097, 31895, 32766, 32355, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65871 expected_overflow=0 exact_recv_per_rank=[33096, 32913, 32636, 32775, 33096, 32510, 32273, 32845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65941 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65839 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 33097, 31895, 32766, 32355, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65369 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65839 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=2 budget=40960 expected_recv=65871 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65411 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 33097, 31895, 32766, 32355, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=40960 expected_recv=63412 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 33097, 31895, 32766, 32355, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65411 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=13 calls=2 budget=40960 expected_recv=63412 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=40960 expected_recv=65639 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 33096, 32331, 33096, 32544, 32462, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 33097, 31895, 32766, 32355, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=14 calls=2 budget=40960 expected_recv=65639 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65187 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 33097, 31895, 32766, 32355, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 33096, 32331, 33096, 32544, 32462, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=40960 expected_recv=65525 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 33096, 32331, 33096, 32544, 32462, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32743, 33096, 33096, 33097, 31895, 32766, 32355, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=2 budget=40960 expected_recv=65187 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=2 budget=40960 expected_recv=65525 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=13 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 33096, 32331, 33096, 32544, 32462, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 33096, 32331, 33096, 32544, 32462, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31365, 32700, 33096, 33096, 32761, 33096, 32934, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=40960 expected_recv=64859 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 33096, 32331, 33096, 32544, 32462, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=40960 expected_recv=64869 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 33096, 32331, 33096, 32544, 32462, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=40960 expected_recv=64924 expected_overflow=0 exact_recv_per_rank=[31365, 32700, 33096, 33096, 32761, 33096, 32934, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=40960 expected_recv=62946 expected_overflow=0 exact_recv_per_rank=[31365, 32700, 33096, 33096, 32761, 33096, 32934, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=40960 expected_recv=64820 expected_overflow=0 exact_recv_per_rank=[32795, 33096, 33096, 32331, 33096, 32544, 32462, 32724] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=2 budget=40960 expected_recv=64859 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=14 calls=2 budget=40960 expected_recv=64869 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=15 calls=2 budget=40960 expected_recv=64924 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=15 calls=2 budget=40960 expected_recv=62946 actual_valid_recv=64000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=14 calls=2 budget=40960 expected_recv=64820 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31365, 32700, 33096, 33096, 32761, 33096, 32934, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=64000 slack_slots=17920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=40960 expected_recv=65619 expected_overflow=0 exact_recv_per_rank=[31365, 32700, 33096, 33096, 32761, 33096, 32934, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=15 calls=2 budget=40960 expected_recv=65619 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[31365, 32700, 33096, 33096, 32761, 33096, 32934, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[40665, 30451, 45081, 10873, 54060, 41872, 20730, 18412] avg=32768.0000 max=54060 max_rank=4 min=10873 min_rank=3 imbalance_ratio=1.649780 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=40960 expected_recv=66030 expected_overflow=0 exact_recv_per_rank=[31365, 32700, 33096, 33096, 32761, 33096, 32934, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=15 calls=2 budget=40960 expected_recv=66030 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[40665, 30451, 45081, 10873, 54060, 41872, 20730, 18412] avg=32768.0000 max=54060 max_rank=4 min=10873 min_rank=3 imbalance_ratio=1.649780 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[31365, 32700, 33096, 33096, 32761, 33096, 32934, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[40665, 30451, 45081, 10873, 54060, 41872, 20730, 18412] avg=32768.0000 max=54060 max_rank=4 min=10873 min_rank=3 imbalance_ratio=1.649780 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=15 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[33095, 31731, 33097, 31837, 33096, 33095, 33098, 33095] avg=32768.0000 max=33098 max_rank=6 min=31731 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20962 max_load_reduction_pct=0.387754 moved_to_replicas=98776 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[40665, 30451, 45081, 10873, 54060, 41872, 20730, 18412] avg=32768.0000 max=54060 max_rank=4 min=10873 min_rank=3 imbalance_ratio=1.649780 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[40665, 30451, 45081, 10873, 54060, 41872, 20730, 18412] avg=32768.0000 max=54060 max_rank=4 min=10873 min_rank=3 imbalance_ratio=1.649780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[33095, 31731, 33097, 31837, 33096, 33095, 33098, 33095] avg=32768.0000 max=33098 max_rank=6 min=31731 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20962 max_load_reduction_pct=0.387754 moved_to_replicas=98776 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[33095, 31731, 33097, 31837, 33096, 33095, 33098, 33095] avg=32768.0000 max=33098 max_rank=6 min=31731 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20962 max_load_reduction_pct=0.387754 moved_to_replicas=98776 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[40665, 30451, 45081, 10873, 54060, 41872, 20730, 18412] avg=32768.0000 max=54060 max_rank=4 min=10873 min_rank=3 imbalance_ratio=1.649780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[40665, 30451, 45081, 10873, 54060, 41872, 20730, 18412] avg=32768.0000 max=54060 max_rank=4 min=10873 min_rank=3 imbalance_ratio=1.649780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[41836, 724, 32059, 31514, 30, 32673, 93371, 29937] avg=32768.0000 max=93371 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.849457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[33095, 31731, 33097, 31837, 33096, 33095, 33098, 33095] avg=32768.0000 max=33098 max_rank=6 min=31731 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20962 max_load_reduction_pct=0.387754 moved_to_replicas=98776 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[33095, 31731, 33097, 31837, 33096, 33095, 33098, 33095] avg=32768.0000 max=33098 max_rank=6 min=31731 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20962 max_load_reduction_pct=0.387754 moved_to_replicas=98776 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[40665, 30451, 45081, 10873, 54060, 41872, 20730, 18412] avg=32768.0000 max=54060 max_rank=4 min=10873 min_rank=3 imbalance_ratio=1.649780 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[41836, 724, 32059, 31514, 30, 32673, 93371, 29937] avg=32768.0000 max=93371 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.849457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[41836, 724, 32059, 31514, 30, 32673, 93371, 29937] avg=32768.0000 max=93371 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.849457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[33097, 33096, 32059, 32250, 32778, 32673, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=32059 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=60274 max_load_reduction_pct=0.645532 moved_to_replicas=137830 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[33095, 31731, 33097, 31837, 33096, 33095, 33098, 33095] avg=32768.0000 max=33098 max_rank=6 min=31731 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20962 max_load_reduction_pct=0.387754 moved_to_replicas=98776 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[33095, 31731, 33097, 31837, 33096, 33095, 33098, 33095] avg=32768.0000 max=33098 max_rank=6 min=31731 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20962 max_load_reduction_pct=0.387754 moved_to_replicas=98776 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[41836, 724, 32059, 31514, 30, 32673, 93371, 29937] avg=32768.0000 max=93371 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.849457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[33097, 33096, 32059, 32250, 32778, 32673, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=32059 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=60274 max_load_reduction_pct=0.645532 moved_to_replicas=137830 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[41836, 724, 32059, 31514, 30, 32673, 93371, 29937] avg=32768.0000 max=93371 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.849457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[33095, 31731, 33097, 31837, 33096, 33095, 33098, 33095] avg=32768.0000 max=33098 max_rank=6 min=31731 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=20962 max_load_reduction_pct=0.387754 moved_to_replicas=98776 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[33097, 33096, 32059, 32250, 32778, 32673, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=32059 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=60274 max_load_reduction_pct=0.645532 moved_to_replicas=137830 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[122, 65155, 30015, 98478, 34972, 971, 32286, 145] avg=32768.0000 max=98478 max_rank=3 min=122 min_rank=0 imbalance_ratio=3.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[41836, 724, 32059, 31514, 30, 32673, 93371, 29937] avg=32768.0000 max=93371 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.849457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[41836, 724, 32059, 31514, 30, 32673, 93371, 29937] avg=32768.0000 max=93371 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.849457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[33097, 33096, 32059, 32250, 32778, 32673, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=32059 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=60274 max_load_reduction_pct=0.645532 moved_to_replicas=137830 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[33097, 33096, 32059, 32250, 32778, 32673, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=32059 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=60274 max_load_reduction_pct=0.645532 moved_to_replicas=137830 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[122, 65155, 30015, 98478, 34972, 971, 32286, 145] avg=32768.0000 max=98478 max_rank=3 min=122 min_rank=0 imbalance_ratio=3.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[41836, 724, 32059, 31514, 30, 32673, 93371, 29937] avg=32768.0000 max=93371 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.849457 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[122, 65155, 30015, 98478, 34972, 971, 32286, 145] avg=32768.0000 max=98478 max_rank=3 min=122 min_rank=0 imbalance_ratio=3.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[32839, 33096, 31891, 33096, 33096, 33046, 32286, 32794] avg=32768.0000 max=33096 max_rank=1 min=31891 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65382 max_load_reduction_pct=0.663925 moved_to_replicas=199474 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[33097, 33096, 32059, 32250, 32778, 32673, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=32059 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=60274 max_load_reduction_pct=0.645532 moved_to_replicas=137830 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[33097, 33096, 32059, 32250, 32778, 32673, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=32059 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=60274 max_load_reduction_pct=0.645532 moved_to_replicas=137830 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[32839, 33096, 31891, 33096, 33096, 33046, 32286, 32794] avg=32768.0000 max=33096 max_rank=1 min=31891 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65382 max_load_reduction_pct=0.663925 moved_to_replicas=199474 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[122, 65155, 30015, 98478, 34972, 971, 32286, 145] avg=32768.0000 max=98478 max_rank=3 min=122 min_rank=0 imbalance_ratio=3.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[33097, 33096, 32059, 32250, 32778, 32673, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=32059 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=60274 max_load_reduction_pct=0.645532 moved_to_replicas=137830 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[32839, 33096, 31891, 33096, 33096, 33046, 32286, 32794] avg=32768.0000 max=33096 max_rank=1 min=31891 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65382 max_load_reduction_pct=0.663925 moved_to_replicas=199474 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[122, 65155, 30015, 98478, 34972, 971, 32286, 145] avg=32768.0000 max=98478 max_rank=3 min=122 min_rank=0 imbalance_ratio=3.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[33739, 1226, 32881, 94421, 2325, 64447, 327, 32778] avg=32768.0000 max=94421 max_rank=3 min=327 min_rank=6 imbalance_ratio=2.881500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[122, 65155, 30015, 98478, 34972, 971, 32286, 145] avg=32768.0000 max=98478 max_rank=3 min=122 min_rank=0 imbalance_ratio=3.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[122, 65155, 30015, 98478, 34972, 971, 32286, 145] avg=32768.0000 max=98478 max_rank=3 min=122 min_rank=0 imbalance_ratio=3.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[32839, 33096, 31891, 33096, 33096, 33046, 32286, 32794] avg=32768.0000 max=33096 max_rank=1 min=31891 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65382 max_load_reduction_pct=0.663925 moved_to_replicas=199474 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[32839, 33096, 31891, 33096, 33096, 33046, 32286, 32794] avg=32768.0000 max=33096 max_rank=1 min=31891 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65382 max_load_reduction_pct=0.663925 moved_to_replicas=199474 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[33739, 1226, 32881, 94421, 2325, 64447, 327, 32778] avg=32768.0000 max=94421 max_rank=3 min=327 min_rank=6 imbalance_ratio=2.881500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[33096, 31065, 32881, 33098, 33096, 33096, 33034, 32778] avg=32768.0000 max=33098 max_rank=3 min=31065 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=61323 max_load_reduction_pct=0.649464 moved_to_replicas=189432 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[122, 65155, 30015, 98478, 34972, 971, 32286, 145] avg=32768.0000 max=98478 max_rank=3 min=122 min_rank=0 imbalance_ratio=3.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[33739, 1226, 32881, 94421, 2325, 64447, 327, 32778] avg=32768.0000 max=94421 max_rank=3 min=327 min_rank=6 imbalance_ratio=2.881500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[32839, 33096, 31891, 33096, 33096, 33046, 32286, 32794] avg=32768.0000 max=33096 max_rank=1 min=31891 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65382 max_load_reduction_pct=0.663925 moved_to_replicas=199474 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[32839, 33096, 31891, 33096, 33096, 33046, 32286, 32794] avg=32768.0000 max=33096 max_rank=1 min=31891 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65382 max_load_reduction_pct=0.663925 moved_to_replicas=199474 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[33096, 31065, 32881, 33098, 33096, 33096, 33034, 32778] avg=32768.0000 max=33098 max_rank=3 min=31065 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=61323 max_load_reduction_pct=0.649464 moved_to_replicas=189432 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[33739, 1226, 32881, 94421, 2325, 64447, 327, 32778] avg=32768.0000 max=94421 max_rank=3 min=327 min_rank=6 imbalance_ratio=2.881500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[33096, 31065, 32881, 33098, 33096, 33096, 33034, 32778] avg=32768.0000 max=33098 max_rank=3 min=31065 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=61323 max_load_reduction_pct=0.649464 moved_to_replicas=189432 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[33739, 1226, 32881, 94421, 2325, 64447, 327, 32778] avg=32768.0000 max=94421 max_rank=3 min=327 min_rank=6 imbalance_ratio=2.881500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[32839, 33096, 31891, 33096, 33096, 33046, 32286, 32794] avg=32768.0000 max=33096 max_rank=1 min=31891 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65382 max_load_reduction_pct=0.663925 moved_to_replicas=199474 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[144, 38341, 37860, 32790, 29251, 32759, 32753, 58246] avg=32768.0000 max=58246 max_rank=7 min=144 min_rank=0 imbalance_ratio=1.777527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[33739, 1226, 32881, 94421, 2325, 64447, 327, 32778] avg=32768.0000 max=94421 max_rank=3 min=327 min_rank=6 imbalance_ratio=2.881500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[33096, 31065, 32881, 33098, 33096, 33096, 33034, 32778] avg=32768.0000 max=33098 max_rank=3 min=31065 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=61323 max_load_reduction_pct=0.649464 moved_to_replicas=189432 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[33739, 1226, 32881, 94421, 2325, 64447, 327, 32778] avg=32768.0000 max=94421 max_rank=3 min=327 min_rank=6 imbalance_ratio=2.881500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[144, 38341, 37860, 32790, 29251, 32759, 32753, 58246] avg=32768.0000 max=58246 max_rank=7 min=144 min_rank=0 imbalance_ratio=1.777527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[33096, 31065, 32881, 33098, 33096, 33096, 33034, 32778] avg=32768.0000 max=33098 max_rank=3 min=31065 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=61323 max_load_reduction_pct=0.649464 moved_to_replicas=189432 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[31459, 33095, 33096, 32790, 33096, 32759, 32753, 33096] avg=32768.0000 max=33096 max_rank=2 min=31459 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25150 max_load_reduction_pct=0.431789 moved_to_replicas=67037 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[144, 38341, 37860, 32790, 29251, 32759, 32753, 58246] avg=32768.0000 max=58246 max_rank=7 min=144 min_rank=0 imbalance_ratio=1.777527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[33739, 1226, 32881, 94421, 2325, 64447, 327, 32778] avg=32768.0000 max=94421 max_rank=3 min=327 min_rank=6 imbalance_ratio=2.881500 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[33096, 31065, 32881, 33098, 33096, 33096, 33034, 32778] avg=32768.0000 max=33098 max_rank=3 min=31065 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=61323 max_load_reduction_pct=0.649464 moved_to_replicas=189432 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[33096, 31065, 32881, 33098, 33096, 33096, 33034, 32778] avg=32768.0000 max=33098 max_rank=3 min=31065 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=61323 max_load_reduction_pct=0.649464 moved_to_replicas=189432 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[31459, 33095, 33096, 32790, 33096, 32759, 32753, 33096] avg=32768.0000 max=33096 max_rank=2 min=31459 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25150 max_load_reduction_pct=0.431789 moved_to_replicas=67037 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[144, 38341, 37860, 32790, 29251, 32759, 32753, 58246] avg=32768.0000 max=58246 max_rank=7 min=144 min_rank=0 imbalance_ratio=1.777527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[31459, 33095, 33096, 32790, 33096, 32759, 32753, 33096] avg=32768.0000 max=33096 max_rank=2 min=31459 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25150 max_load_reduction_pct=0.431789 moved_to_replicas=67037 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[144, 38341, 37860, 32790, 29251, 32759, 32753, 58246] avg=32768.0000 max=58246 max_rank=7 min=144 min_rank=0 imbalance_ratio=1.777527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[32794, 32718, 885, 109, 32859, 33763, 65361, 63655] avg=32768.0000 max=65361 max_rank=6 min=109 min_rank=3 imbalance_ratio=1.994659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[33096, 31065, 32881, 33098, 33096, 33096, 33034, 32778] avg=32768.0000 max=33098 max_rank=3 min=31065 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=61323 max_load_reduction_pct=0.649464 moved_to_replicas=189432 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[144, 38341, 37860, 32790, 29251, 32759, 32753, 58246] avg=32768.0000 max=58246 max_rank=7 min=144 min_rank=0 imbalance_ratio=1.777527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[31459, 33095, 33096, 32790, 33096, 32759, 32753, 33096] avg=32768.0000 max=33096 max_rank=2 min=31459 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25150 max_load_reduction_pct=0.431789 moved_to_replicas=67037 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[32794, 32718, 885, 109, 32859, 33763, 65361, 63655] avg=32768.0000 max=65361 max_rank=6 min=109 min_rank=3 imbalance_ratio=1.994659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[144, 38341, 37860, 32790, 29251, 32759, 32753, 58246] avg=32768.0000 max=58246 max_rank=7 min=144 min_rank=0 imbalance_ratio=1.777527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[31459, 33095, 33096, 32790, 33096, 32759, 32753, 33096] avg=32768.0000 max=33096 max_rank=2 min=31459 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25150 max_load_reduction_pct=0.431789 moved_to_replicas=67037 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[32794, 32718, 32111, 32374, 32859, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32265 max_load_reduction_pct=0.493643 moved_to_replicas=127604 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[32794, 32718, 885, 109, 32859, 33763, 65361, 63655] avg=32768.0000 max=65361 max_rank=6 min=109 min_rank=3 imbalance_ratio=1.994659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[144, 38341, 37860, 32790, 29251, 32759, 32753, 58246] avg=32768.0000 max=58246 max_rank=7 min=144 min_rank=0 imbalance_ratio=1.777527 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[31459, 33095, 33096, 32790, 33096, 32759, 32753, 33096] avg=32768.0000 max=33096 max_rank=2 min=31459 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25150 max_load_reduction_pct=0.431789 moved_to_replicas=67037 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[32794, 32718, 32111, 32374, 32859, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32265 max_load_reduction_pct=0.493643 moved_to_replicas=127604 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[31459, 33095, 33096, 32790, 33096, 32759, 32753, 33096] avg=32768.0000 max=33096 max_rank=2 min=31459 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25150 max_load_reduction_pct=0.431789 moved_to_replicas=67037 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[32794, 32718, 885, 109, 32859, 33763, 65361, 63655] avg=32768.0000 max=65361 max_rank=6 min=109 min_rank=3 imbalance_ratio=1.994659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[32794, 32718, 32111, 32374, 32859, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32265 max_load_reduction_pct=0.493643 moved_to_replicas=127604 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[32794, 32718, 885, 109, 32859, 33763, 65361, 63655] avg=32768.0000 max=65361 max_rank=6 min=109 min_rank=3 imbalance_ratio=1.994659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[31459, 33095, 33096, 32790, 33096, 32759, 32753, 33096] avg=32768.0000 max=33096 max_rank=2 min=31459 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=25150 max_load_reduction_pct=0.431789 moved_to_replicas=67037 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[10949, 32967, 32772, 32804, 5083, 90443, 24346, 32780] avg=32768.0000 max=90443 max_rank=5 min=5083 min_rank=4 imbalance_ratio=2.760101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[32794, 32718, 885, 109, 32859, 33763, 65361, 63655] avg=32768.0000 max=65361 max_rank=6 min=109 min_rank=3 imbalance_ratio=1.994659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[32794, 32718, 32111, 32374, 32859, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32265 max_load_reduction_pct=0.493643 moved_to_replicas=127604 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[10949, 32967, 32772, 32804, 5083, 90443, 24346, 32780] avg=32768.0000 max=90443 max_rank=5 min=5083 min_rank=4 imbalance_ratio=2.760101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[32794, 32718, 32111, 32374, 32859, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32265 max_load_reduction_pct=0.493643 moved_to_replicas=127604 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[32794, 32718, 885, 109, 32859, 33763, 65361, 63655] avg=32768.0000 max=65361 max_rank=6 min=109 min_rank=3 imbalance_ratio=1.994659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[33095, 32967, 32772, 32804, 33098, 33094, 31534, 32780] avg=32768.0000 max=33098 max_rank=4 min=31534 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=57345 max_load_reduction_pct=0.634046 moved_to_replicas=117452 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[10949, 32967, 32772, 32804, 5083, 90443, 24346, 32780] avg=32768.0000 max=90443 max_rank=5 min=5083 min_rank=4 imbalance_ratio=2.760101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[32794, 32718, 32111, 32374, 32859, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32265 max_load_reduction_pct=0.493643 moved_to_replicas=127604 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[32794, 32718, 885, 109, 32859, 33763, 65361, 63655] avg=32768.0000 max=65361 max_rank=6 min=109 min_rank=3 imbalance_ratio=1.994659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[33095, 32967, 32772, 32804, 33098, 33094, 31534, 32780] avg=32768.0000 max=33098 max_rank=4 min=31534 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=57345 max_load_reduction_pct=0.634046 moved_to_replicas=117452 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[32794, 32718, 32111, 32374, 32859, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32265 max_load_reduction_pct=0.493643 moved_to_replicas=127604 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[10949, 32967, 32772, 32804, 5083, 90443, 24346, 32780] avg=32768.0000 max=90443 max_rank=5 min=5083 min_rank=4 imbalance_ratio=2.760101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[33095, 32967, 32772, 32804, 33098, 33094, 31534, 32780] avg=32768.0000 max=33098 max_rank=4 min=31534 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=57345 max_load_reduction_pct=0.634046 moved_to_replicas=117452 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[10949, 32967, 32772, 32804, 5083, 90443, 24346, 32780] avg=32768.0000 max=90443 max_rank=5 min=5083 min_rank=4 imbalance_ratio=2.760101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[62928, 3662, 38405, 33311, 54319, 838, 64721, 3960] avg=32768.0000 max=64721 max_rank=6 min=838 min_rank=5 imbalance_ratio=1.975128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[32794, 32718, 32111, 32374, 32859, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=5 min=32111 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32265 max_load_reduction_pct=0.493643 moved_to_replicas=127604 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[10949, 32967, 32772, 32804, 5083, 90443, 24346, 32780] avg=32768.0000 max=90443 max_rank=5 min=5083 min_rank=4 imbalance_ratio=2.760101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[33095, 32967, 32772, 32804, 33098, 33094, 31534, 32780] avg=32768.0000 max=33098 max_rank=4 min=31534 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=57345 max_load_reduction_pct=0.634046 moved_to_replicas=117452 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[62928, 3662, 38405, 33311, 54319, 838, 64721, 3960] avg=32768.0000 max=64721 max_rank=6 min=838 min_rank=5 imbalance_ratio=1.975128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[33095, 32967, 32772, 32804, 33098, 33094, 31534, 32780] avg=32768.0000 max=33098 max_rank=4 min=31534 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=57345 max_load_reduction_pct=0.634046 moved_to_replicas=117452 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[10949, 32967, 32772, 32804, 5083, 90443, 24346, 32780] avg=32768.0000 max=90443 max_rank=5 min=5083 min_rank=4 imbalance_ratio=2.760101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 32463, 33096, 31105] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31625 max_load_reduction_pct=0.488636 moved_to_replicas=169791 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[62928, 3662, 38405, 33311, 54319, 838, 64721, 3960] avg=32768.0000 max=64721 max_rank=6 min=838 min_rank=5 imbalance_ratio=1.975128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[33095, 32967, 32772, 32804, 33098, 33094, 31534, 32780] avg=32768.0000 max=33098 max_rank=4 min=31534 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=57345 max_load_reduction_pct=0.634046 moved_to_replicas=117452 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[10949, 32967, 32772, 32804, 5083, 90443, 24346, 32780] avg=32768.0000 max=90443 max_rank=5 min=5083 min_rank=4 imbalance_ratio=2.760101 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 32463, 33096, 31105] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31625 max_load_reduction_pct=0.488636 moved_to_replicas=169791 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[33095, 32967, 32772, 32804, 33098, 33094, 31534, 32780] avg=32768.0000 max=33098 max_rank=4 min=31534 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=57345 max_load_reduction_pct=0.634046 moved_to_replicas=117452 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[62928, 3662, 38405, 33311, 54319, 838, 64721, 3960] avg=32768.0000 max=64721 max_rank=6 min=838 min_rank=5 imbalance_ratio=1.975128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 32463, 33096, 31105] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31625 max_load_reduction_pct=0.488636 moved_to_replicas=169791 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[62928, 3662, 38405, 33311, 54319, 838, 64721, 3960] avg=32768.0000 max=64721 max_rank=6 min=838 min_rank=5 imbalance_ratio=1.975128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[66207, 55, 33445, 32786, 65449, 25937, 5974, 32291] avg=32768.0000 max=66207 max_rank=0 min=55 min_rank=1 imbalance_ratio=2.020477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[33095, 32967, 32772, 32804, 33098, 33094, 31534, 32780] avg=32768.0000 max=33098 max_rank=4 min=31534 min_rank=6 imbalance_ratio=1.010071 max_load_reduction=57345 max_load_reduction_pct=0.634046 moved_to_replicas=117452 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[62928, 3662, 38405, 33311, 54319, 838, 64721, 3960] avg=32768.0000 max=64721 max_rank=6 min=838 min_rank=5 imbalance_ratio=1.975128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[66207, 55, 33445, 32786, 65449, 25937, 5974, 32291] avg=32768.0000 max=66207 max_rank=0 min=55 min_rank=1 imbalance_ratio=2.020477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 32463, 33096, 31105] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31625 max_load_reduction_pct=0.488636 moved_to_replicas=169791 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 32463, 33096, 31105] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31625 max_load_reduction_pct=0.488636 moved_to_replicas=169791 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32781, 33096, 32786, 33096, 31903, 33095, 32291] avg=32768.0000 max=33096 max_rank=0 min=31903 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33111 max_load_reduction_pct=0.500113 moved_to_replicas=131737 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[62928, 3662, 38405, 33311, 54319, 838, 64721, 3960] avg=32768.0000 max=64721 max_rank=6 min=838 min_rank=5 imbalance_ratio=1.975128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[66207, 55, 33445, 32786, 65449, 25937, 5974, 32291] avg=32768.0000 max=66207 max_rank=0 min=55 min_rank=1 imbalance_ratio=2.020477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 32463, 33096, 31105] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31625 max_load_reduction_pct=0.488636 moved_to_replicas=169791 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[62928, 3662, 38405, 33311, 54319, 838, 64721, 3960] avg=32768.0000 max=64721 max_rank=6 min=838 min_rank=5 imbalance_ratio=1.975128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32781, 33096, 32786, 33096, 31903, 33095, 32291] avg=32768.0000 max=33096 max_rank=0 min=31903 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33111 max_load_reduction_pct=0.500113 moved_to_replicas=131737 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 32463, 33096, 31105] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31625 max_load_reduction_pct=0.488636 moved_to_replicas=169791 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[66207, 55, 33445, 32786, 65449, 25937, 5974, 32291] avg=32768.0000 max=66207 max_rank=0 min=55 min_rank=1 imbalance_ratio=2.020477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[97804, 7278, 32560, 32658, 32976, 45, 32977, 25846] avg=32768.0000 max=97804 max_rank=0 min=45 min_rank=5 imbalance_ratio=2.984741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32781, 33096, 32786, 33096, 31903, 33095, 32291] avg=32768.0000 max=33096 max_rank=0 min=31903 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33111 max_load_reduction_pct=0.500113 moved_to_replicas=131737 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[66207, 55, 33445, 32786, 65449, 25937, 5974, 32291] avg=32768.0000 max=66207 max_rank=0 min=55 min_rank=1 imbalance_ratio=2.020477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 32463, 33096, 31105] avg=32768.0000 max=33096 max_rank=0 min=31105 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31625 max_load_reduction_pct=0.488636 moved_to_replicas=169791 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[66207, 55, 33445, 32786, 65449, 25937, 5974, 32291] avg=32768.0000 max=66207 max_rank=0 min=55 min_rank=1 imbalance_ratio=2.020477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[97804, 7278, 32560, 32658, 32976, 45, 32977, 25846] avg=32768.0000 max=97804 max_rank=0 min=45 min_rank=5 imbalance_ratio=2.984741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32781, 33096, 32786, 33096, 31903, 33095, 32291] avg=32768.0000 max=33096 max_rank=0 min=31903 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33111 max_load_reduction_pct=0.500113 moved_to_replicas=131737 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[33096, 33096, 32560, 32658, 32976, 32755, 32977, 32026] avg=32768.0000 max=33096 max_rank=0 min=32026 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64708 max_load_reduction_pct=0.661609 moved_to_replicas=128423 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32781, 33096, 32786, 33096, 31903, 33095, 32291] avg=32768.0000 max=33096 max_rank=0 min=31903 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33111 max_load_reduction_pct=0.500113 moved_to_replicas=131737 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[66207, 55, 33445, 32786, 65449, 25937, 5974, 32291] avg=32768.0000 max=66207 max_rank=0 min=55 min_rank=1 imbalance_ratio=2.020477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[97804, 7278, 32560, 32658, 32976, 45, 32977, 25846] avg=32768.0000 max=97804 max_rank=0 min=45 min_rank=5 imbalance_ratio=2.984741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32781, 33096, 32786, 33096, 31903, 33095, 32291] avg=32768.0000 max=33096 max_rank=0 min=31903 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33111 max_load_reduction_pct=0.500113 moved_to_replicas=131737 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[66207, 55, 33445, 32786, 65449, 25937, 5974, 32291] avg=32768.0000 max=66207 max_rank=0 min=55 min_rank=1 imbalance_ratio=2.020477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[33096, 33096, 32560, 32658, 32976, 32755, 32977, 32026] avg=32768.0000 max=33096 max_rank=0 min=32026 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64708 max_load_reduction_pct=0.661609 moved_to_replicas=128423 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32781, 33096, 32786, 33096, 31903, 33095, 32291] avg=32768.0000 max=33096 max_rank=0 min=31903 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33111 max_load_reduction_pct=0.500113 moved_to_replicas=131737 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[97804, 7278, 32560, 32658, 32976, 45, 32977, 25846] avg=32768.0000 max=97804 max_rank=0 min=45 min_rank=5 imbalance_ratio=2.984741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[135, 32710, 65248, 688, 27162, 32801, 37989, 65411] avg=32768.0000 max=65411 max_rank=7 min=135 min_rank=0 imbalance_ratio=1.996185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[33096, 33096, 32560, 32658, 32976, 32755, 32977, 32026] avg=32768.0000 max=33096 max_rank=0 min=32026 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64708 max_load_reduction_pct=0.661609 moved_to_replicas=128423 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[97804, 7278, 32560, 32658, 32976, 45, 32977, 25846] avg=32768.0000 max=97804 max_rank=0 min=45 min_rank=5 imbalance_ratio=2.984741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[33096, 32781, 33096, 32786, 33096, 31903, 33095, 32291] avg=32768.0000 max=33096 max_rank=0 min=31903 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33111 max_load_reduction_pct=0.500113 moved_to_replicas=131737 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[97804, 7278, 32560, 32658, 32976, 45, 32977, 25846] avg=32768.0000 max=97804 max_rank=0 min=45 min_rank=5 imbalance_ratio=2.984741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[135, 32710, 65248, 688, 27162, 32801, 37989, 65411] avg=32768.0000 max=65411 max_rank=7 min=135 min_rank=0 imbalance_ratio=1.996185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[33096, 33096, 32560, 32658, 32976, 32755, 32977, 32026] avg=32768.0000 max=33096 max_rank=0 min=32026 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64708 max_load_reduction_pct=0.661609 moved_to_replicas=128423 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[32450, 32710, 33096, 32840, 32054, 32801, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=32054 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32314 max_load_reduction_pct=0.494015 moved_to_replicas=134376 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[33096, 33096, 32560, 32658, 32976, 32755, 32977, 32026] avg=32768.0000 max=33096 max_rank=0 min=32026 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64708 max_load_reduction_pct=0.661609 moved_to_replicas=128423 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[97804, 7278, 32560, 32658, 32976, 45, 32977, 25846] avg=32768.0000 max=97804 max_rank=0 min=45 min_rank=5 imbalance_ratio=2.984741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[135, 32710, 65248, 688, 27162, 32801, 37989, 65411] avg=32768.0000 max=65411 max_rank=7 min=135 min_rank=0 imbalance_ratio=1.996185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[33096, 33096, 32560, 32658, 32976, 32755, 32977, 32026] avg=32768.0000 max=33096 max_rank=0 min=32026 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64708 max_load_reduction_pct=0.661609 moved_to_replicas=128423 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[32450, 32710, 33096, 32840, 32054, 32801, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=32054 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32314 max_load_reduction_pct=0.494015 moved_to_replicas=134376 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[97804, 7278, 32560, 32658, 32976, 45, 32977, 25846] avg=32768.0000 max=97804 max_rank=0 min=45 min_rank=5 imbalance_ratio=2.984741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[33096, 33096, 32560, 32658, 32976, 32755, 32977, 32026] avg=32768.0000 max=33096 max_rank=0 min=32026 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64708 max_load_reduction_pct=0.661609 moved_to_replicas=128423 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[131, 127, 97366, 13976, 34737, 32730, 49424, 33653] avg=32768.0000 max=97366 max_rank=2 min=127 min_rank=1 imbalance_ratio=2.971375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[135, 32710, 65248, 688, 27162, 32801, 37989, 65411] avg=32768.0000 max=65411 max_rank=7 min=135 min_rank=0 imbalance_ratio=1.996185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[32450, 32710, 33096, 32840, 32054, 32801, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=32054 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32314 max_load_reduction_pct=0.494015 moved_to_replicas=134376 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[135, 32710, 65248, 688, 27162, 32801, 37989, 65411] avg=32768.0000 max=65411 max_rank=7 min=135 min_rank=0 imbalance_ratio=1.996185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[33096, 33096, 32560, 32658, 32976, 32755, 32977, 32026] avg=32768.0000 max=33096 max_rank=0 min=32026 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64708 max_load_reduction_pct=0.661609 moved_to_replicas=128423 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[135, 32710, 65248, 688, 27162, 32801, 37989, 65411] avg=32768.0000 max=65411 max_rank=7 min=135 min_rank=0 imbalance_ratio=1.996185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[131, 127, 97366, 13976, 34737, 32730, 49424, 33653] avg=32768.0000 max=97366 max_rank=2 min=127 min_rank=1 imbalance_ratio=2.971375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[32252, 32833, 33096, 31944, 33096, 32730, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31944 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64269 max_load_reduction_pct=0.660076 moved_to_replicas=161507 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[32450, 32710, 33096, 32840, 32054, 32801, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=32054 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32314 max_load_reduction_pct=0.494015 moved_to_replicas=134376 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[32450, 32710, 33096, 32840, 32054, 32801, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=32054 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32314 max_load_reduction_pct=0.494015 moved_to_replicas=134376 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[135, 32710, 65248, 688, 27162, 32801, 37989, 65411] avg=32768.0000 max=65411 max_rank=7 min=135 min_rank=0 imbalance_ratio=1.996185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[32450, 32710, 33096, 32840, 32054, 32801, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=32054 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32314 max_load_reduction_pct=0.494015 moved_to_replicas=134376 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[131, 127, 97366, 13976, 34737, 32730, 49424, 33653] avg=32768.0000 max=97366 max_rank=2 min=127 min_rank=1 imbalance_ratio=2.971375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[32252, 32833, 33096, 31944, 33096, 32730, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31944 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64269 max_load_reduction_pct=0.660076 moved_to_replicas=161507 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[135, 32710, 65248, 688, 27162, 32801, 37989, 65411] avg=32768.0000 max=65411 max_rank=7 min=135 min_rank=0 imbalance_ratio=1.996185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[65335, 32913, 32636, 437, 65924, 32510, 32273, 116] avg=32768.0000 max=65924 max_rank=4 min=116 min_rank=7 imbalance_ratio=2.011841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[32450, 32710, 33096, 32840, 32054, 32801, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=32054 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32314 max_load_reduction_pct=0.494015 moved_to_replicas=134376 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[131, 127, 97366, 13976, 34737, 32730, 49424, 33653] avg=32768.0000 max=97366 max_rank=2 min=127 min_rank=1 imbalance_ratio=2.971375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[131, 127, 97366, 13976, 34737, 32730, 49424, 33653] avg=32768.0000 max=97366 max_rank=2 min=127 min_rank=1 imbalance_ratio=2.971375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[32252, 32833, 33096, 31944, 33096, 32730, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31944 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64269 max_load_reduction_pct=0.660076 moved_to_replicas=161507 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[32450, 32710, 33096, 32840, 32054, 32801, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=32054 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32314 max_load_reduction_pct=0.494015 moved_to_replicas=134376 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[131, 127, 97366, 13976, 34737, 32730, 49424, 33653] avg=32768.0000 max=97366 max_rank=2 min=127 min_rank=1 imbalance_ratio=2.971375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[33096, 32913, 32636, 32775, 33096, 32510, 32273, 32845] avg=32768.0000 max=33096 max_rank=0 min=32273 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32828 max_load_reduction_pct=0.497967 moved_to_replicas=133282 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[65335, 32913, 32636, 437, 65924, 32510, 32273, 116] avg=32768.0000 max=65924 max_rank=4 min=116 min_rank=7 imbalance_ratio=2.011841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[32252, 32833, 33096, 31944, 33096, 32730, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31944 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64269 max_load_reduction_pct=0.660076 moved_to_replicas=161507 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[32252, 32833, 33096, 31944, 33096, 32730, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31944 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64269 max_load_reduction_pct=0.660076 moved_to_replicas=161507 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[131, 127, 97366, 13976, 34737, 32730, 49424, 33653] avg=32768.0000 max=97366 max_rank=2 min=127 min_rank=1 imbalance_ratio=2.971375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[32252, 32833, 33096, 31944, 33096, 32730, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31944 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64269 max_load_reduction_pct=0.660076 moved_to_replicas=161507 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[65335, 32913, 32636, 437, 65924, 32510, 32273, 116] avg=32768.0000 max=65924 max_rank=4 min=116 min_rank=7 imbalance_ratio=2.011841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[33096, 32913, 32636, 32775, 33096, 32510, 32273, 32845] avg=32768.0000 max=33096 max_rank=0 min=32273 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32828 max_load_reduction_pct=0.497967 moved_to_replicas=133282 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[131, 127, 97366, 13976, 34737, 32730, 49424, 33653] avg=32768.0000 max=97366 max_rank=2 min=127 min_rank=1 imbalance_ratio=2.971375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[17860, 65243, 33111, 47980, 31880, 619, 8, 65443] avg=32768.0000 max=65443 max_rank=7 min=8 min_rank=6 imbalance_ratio=1.997162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[65335, 32913, 32636, 437, 65924, 32510, 32273, 116] avg=32768.0000 max=65924 max_rank=4 min=116 min_rank=7 imbalance_ratio=2.011841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[32252, 32833, 33096, 31944, 33096, 32730, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31944 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64269 max_load_reduction_pct=0.660076 moved_to_replicas=161507 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[33096, 32913, 32636, 32775, 33096, 32510, 32273, 32845] avg=32768.0000 max=33096 max_rank=0 min=32273 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32828 max_load_reduction_pct=0.497967 moved_to_replicas=133282 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[65335, 32913, 32636, 437, 65924, 32510, 32273, 116] avg=32768.0000 max=65924 max_rank=4 min=116 min_rank=7 imbalance_ratio=2.011841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[32252, 32833, 33096, 31944, 33096, 32730, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31944 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=64269 max_load_reduction_pct=0.660076 moved_to_replicas=161507 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[32743, 33096, 33096, 33097, 31895, 32766, 32355, 33096] avg=32768.0000 max=33097 max_rank=3 min=31895 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32346 max_load_reduction_pct=0.494262 moved_to_replicas=158368 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[65335, 32913, 32636, 437, 65924, 32510, 32273, 116] avg=32768.0000 max=65924 max_rank=4 min=116 min_rank=7 imbalance_ratio=2.011841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[17860, 65243, 33111, 47980, 31880, 619, 8, 65443] avg=32768.0000 max=65443 max_rank=7 min=8 min_rank=6 imbalance_ratio=1.997162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[33096, 32913, 32636, 32775, 33096, 32510, 32273, 32845] avg=32768.0000 max=33096 max_rank=0 min=32273 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32828 max_load_reduction_pct=0.497967 moved_to_replicas=133282 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[33096, 32913, 32636, 32775, 33096, 32510, 32273, 32845] avg=32768.0000 max=33096 max_rank=0 min=32273 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32828 max_load_reduction_pct=0.497967 moved_to_replicas=133282 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[65335, 32913, 32636, 437, 65924, 32510, 32273, 116] avg=32768.0000 max=65924 max_rank=4 min=116 min_rank=7 imbalance_ratio=2.011841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[17860, 65243, 33111, 47980, 31880, 619, 8, 65443] avg=32768.0000 max=65443 max_rank=7 min=8 min_rank=6 imbalance_ratio=1.997162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[33096, 32913, 32636, 32775, 33096, 32510, 32273, 32845] avg=32768.0000 max=33096 max_rank=0 min=32273 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32828 max_load_reduction_pct=0.497967 moved_to_replicas=133282 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[32743, 33096, 33096, 33097, 31895, 32766, 32355, 33096] avg=32768.0000 max=33097 max_rank=3 min=31895 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32346 max_load_reduction_pct=0.494262 moved_to_replicas=158368 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[65335, 32913, 32636, 437, 65924, 32510, 32273, 116] avg=32768.0000 max=65924 max_rank=4 min=116 min_rank=7 imbalance_ratio=2.011841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[32795, 65391, 65371, 31890, 33537, 269, 167, 32724] avg=32768.0000 max=65391 max_rank=1 min=167 min_rank=6 imbalance_ratio=1.995575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[17860, 65243, 33111, 47980, 31880, 619, 8, 65443] avg=32768.0000 max=65443 max_rank=7 min=8 min_rank=6 imbalance_ratio=1.997162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[33096, 32913, 32636, 32775, 33096, 32510, 32273, 32845] avg=32768.0000 max=33096 max_rank=0 min=32273 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32828 max_load_reduction_pct=0.497967 moved_to_replicas=133282 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[32743, 33096, 33096, 33097, 31895, 32766, 32355, 33096] avg=32768.0000 max=33097 max_rank=3 min=31895 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32346 max_load_reduction_pct=0.494262 moved_to_replicas=158368 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[17860, 65243, 33111, 47980, 31880, 619, 8, 65443] avg=32768.0000 max=65443 max_rank=7 min=8 min_rank=6 imbalance_ratio=1.997162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[33096, 32913, 32636, 32775, 33096, 32510, 32273, 32845] avg=32768.0000 max=33096 max_rank=0 min=32273 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32828 max_load_reduction_pct=0.497967 moved_to_replicas=133282 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[32795, 33096, 33096, 32331, 33096, 32544, 32462, 32724] avg=32768.0000 max=33096 max_rank=1 min=32331 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32295 max_load_reduction_pct=0.493875 moved_to_replicas=128879 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[17860, 65243, 33111, 47980, 31880, 619, 8, 65443] avg=32768.0000 max=65443 max_rank=7 min=8 min_rank=6 imbalance_ratio=1.997162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[32795, 65391, 65371, 31890, 33537, 269, 167, 32724] avg=32768.0000 max=65391 max_rank=1 min=167 min_rank=6 imbalance_ratio=1.995575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[32743, 33096, 33096, 33097, 31895, 32766, 32355, 33096] avg=32768.0000 max=33097 max_rank=3 min=31895 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32346 max_load_reduction_pct=0.494262 moved_to_replicas=158368 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[32743, 33096, 33096, 33097, 31895, 32766, 32355, 33096] avg=32768.0000 max=33097 max_rank=3 min=31895 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32346 max_load_reduction_pct=0.494262 moved_to_replicas=158368 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[17860, 65243, 33111, 47980, 31880, 619, 8, 65443] avg=32768.0000 max=65443 max_rank=7 min=8 min_rank=6 imbalance_ratio=1.997162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[32743, 33096, 33096, 33097, 31895, 32766, 32355, 33096] avg=32768.0000 max=33097 max_rank=3 min=31895 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32346 max_load_reduction_pct=0.494262 moved_to_replicas=158368 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[32795, 33096, 33096, 32331, 33096, 32544, 32462, 32724] avg=32768.0000 max=33096 max_rank=1 min=32331 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32295 max_load_reduction_pct=0.493875 moved_to_replicas=128879 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[32795, 65391, 65371, 31890, 33537, 269, 167, 32724] avg=32768.0000 max=65391 max_rank=1 min=167 min_rank=6 imbalance_ratio=1.995575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[30527, 32700, 65198, 65830, 53, 33213, 32934, 1689] avg=32768.0000 max=65830 max_rank=3 min=53 min_rank=4 imbalance_ratio=2.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[17860, 65243, 33111, 47980, 31880, 619, 8, 65443] avg=32768.0000 max=65443 max_rank=7 min=8 min_rank=6 imbalance_ratio=1.997162 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[32795, 65391, 65371, 31890, 33537, 269, 167, 32724] avg=32768.0000 max=65391 max_rank=1 min=167 min_rank=6 imbalance_ratio=1.995575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[32743, 33096, 33096, 33097, 31895, 32766, 32355, 33096] avg=32768.0000 max=33097 max_rank=3 min=31895 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32346 max_load_reduction_pct=0.494262 moved_to_replicas=158368 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[32795, 65391, 65371, 31890, 33537, 269, 167, 32724] avg=32768.0000 max=65391 max_rank=1 min=167 min_rank=6 imbalance_ratio=1.995575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[32795, 33096, 33096, 32331, 33096, 32544, 32462, 32724] avg=32768.0000 max=33096 max_rank=1 min=32331 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32295 max_load_reduction_pct=0.493875 moved_to_replicas=128879 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[31365, 32700, 33096, 33096, 32761, 33096, 32934, 33096] avg=32768.0000 max=33096 max_rank=2 min=31365 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32734 max_load_reduction_pct=0.497250 moved_to_replicas=133338 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[32743, 33096, 33096, 33097, 31895, 32766, 32355, 33096] avg=32768.0000 max=33097 max_rank=3 min=31895 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32346 max_load_reduction_pct=0.494262 moved_to_replicas=158368 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[32795, 65391, 65371, 31890, 33537, 269, 167, 32724] avg=32768.0000 max=65391 max_rank=1 min=167 min_rank=6 imbalance_ratio=1.995575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[32795, 33096, 33096, 32331, 33096, 32544, 32462, 32724] avg=32768.0000 max=33096 max_rank=1 min=32331 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32295 max_load_reduction_pct=0.493875 moved_to_replicas=128879 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[30527, 32700, 65198, 65830, 53, 33213, 32934, 1689] avg=32768.0000 max=65830 max_rank=3 min=53 min_rank=4 imbalance_ratio=2.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[32795, 33096, 33096, 32331, 33096, 32544, 32462, 32724] avg=32768.0000 max=33096 max_rank=1 min=32331 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32295 max_load_reduction_pct=0.493875 moved_to_replicas=128879 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[32795, 65391, 65371, 31890, 33537, 269, 167, 32724] avg=32768.0000 max=65391 max_rank=1 min=167 min_rank=6 imbalance_ratio=1.995575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[32795, 33096, 33096, 32331, 33096, 32544, 32462, 32724] avg=32768.0000 max=33096 max_rank=1 min=32331 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32295 max_load_reduction_pct=0.493875 moved_to_replicas=128879 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[31365, 32700, 33096, 33096, 32761, 33096, 32934, 33096] avg=32768.0000 max=33096 max_rank=2 min=31365 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32734 max_load_reduction_pct=0.497250 moved_to_replicas=133338 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[30527, 32700, 65198, 65830, 53, 33213, 32934, 1689] avg=32768.0000 max=65830 max_rank=3 min=53 min_rank=4 imbalance_ratio=2.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[32795, 65391, 65371, 31890, 33537, 269, 167, 32724] avg=32768.0000 max=65391 max_rank=1 min=167 min_rank=6 imbalance_ratio=1.995575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[30527, 32700, 65198, 65830, 53, 33213, 32934, 1689] avg=32768.0000 max=65830 max_rank=3 min=53 min_rank=4 imbalance_ratio=2.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[32795, 33096, 33096, 32331, 33096, 32544, 32462, 32724] avg=32768.0000 max=33096 max_rank=1 min=32331 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32295 max_load_reduction_pct=0.493875 moved_to_replicas=128879 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[30527, 32700, 65198, 65830, 53, 33213, 32934, 1689] avg=32768.0000 max=65830 max_rank=3 min=53 min_rank=4 imbalance_ratio=2.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[31365, 32700, 33096, 33096, 32761, 33096, 32934, 33096] avg=32768.0000 max=33096 max_rank=2 min=31365 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32734 max_load_reduction_pct=0.497250 moved_to_replicas=133338 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[32795, 33096, 33096, 32331, 33096, 32544, 32462, 32724] avg=32768.0000 max=33096 max_rank=1 min=32331 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32295 max_load_reduction_pct=0.493875 moved_to_replicas=128879 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[30527, 32700, 65198, 65830, 53, 33213, 32934, 1689] avg=32768.0000 max=65830 max_rank=3 min=53 min_rank=4 imbalance_ratio=2.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[31365, 32700, 33096, 33096, 32761, 33096, 32934, 33096] avg=32768.0000 max=33096 max_rank=2 min=31365 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32734 max_load_reduction_pct=0.497250 moved_to_replicas=133338 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[31365, 32700, 33096, 33096, 32761, 33096, 32934, 33096] avg=32768.0000 max=33096 max_rank=2 min=31365 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32734 max_load_reduction_pct=0.497250 moved_to_replicas=133338 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[30527, 32700, 65198, 65830, 53, 33213, 32934, 1689] avg=32768.0000 max=65830 max_rank=3 min=53 min_rank=4 imbalance_ratio=2.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[31365, 32700, 33096, 33096, 32761, 33096, 32934, 33096] avg=32768.0000 max=33096 max_rank=2 min=31365 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32734 max_load_reduction_pct=0.497250 moved_to_replicas=133338 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[30527, 32700, 65198, 65830, 53, 33213, 32934, 1689] avg=32768.0000 max=65830 max_rank=3 min=53 min_rank=4 imbalance_ratio=2.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[31365, 32700, 33096, 33096, 32761, 33096, 32934, 33096] avg=32768.0000 max=33096 max_rank=2 min=31365 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32734 max_load_reduction_pct=0.497250 moved_to_replicas=133338 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[31365, 32700, 33096, 33096, 32761, 33096, 32934, 33096] avg=32768.0000 max=33096 max_rank=2 min=31365 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32734 max_load_reduction_pct=0.497250 moved_to_replicas=133338 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=32 total_expected_recv=1048048 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=250240 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=32 total_expected_recv=1050839 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=247936 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=32 total_expected_recv=1048574 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=250112 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=32 total_expected_recv=1049671 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=249472 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=32 total_expected_recv=1050638 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=249600 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=32 total_expected_recv=1048958 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=249472 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=32 total_expected_recv=1043429 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=255488 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=32 total_expected_recv=1048451 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=249856 peak_demand=33098 required_capacity=1.010071 + [2026-08-30 05:10:28] iteration 4/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 571.9 | throughput per GPU (TFLOP/s/GPU): 583.0 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.322899E+01 | loss scale: 1.0 | grad norm: 31.661 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=40960 expected_recv=61641 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=40960 expected_recv=65801 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=40960 expected_recv=65885 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=2 budget=40960 expected_recv=61641 actual_valid_recv=62336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=62336 slack_slots=19584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=2 budget=40960 expected_recv=65801 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=2 budget=40960 expected_recv=65885 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31170, 32791, 32704, 33097, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31170, 32791, 32704, 33097, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31170, 32791, 32704, 33097, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=40960 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31170, 32791, 32704, 33097, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=40960 expected_recv=64265 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31170, 32791, 32704, 33097, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31170, 32791, 32704, 33097, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=2 budget=40960 expected_recv=65530 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=40960 expected_recv=64416 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31170, 32791, 32704, 33097, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=40960 expected_recv=65308 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31170, 32791, 32704, 33097, 33095, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=2 budget=40960 expected_recv=64265 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=2 budget=40960 expected_recv=64416 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=2 budget=40960 expected_recv=65308 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=40960 expected_recv=63474 expected_overflow=0 exact_recv_per_rank=[31833, 32856, 32702, 33096, 32368, 33095, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=40960 expected_recv=65654 expected_overflow=0 exact_recv_per_rank=[31833, 32856, 32702, 33096, 32368, 33095, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=2 budget=40960 expected_recv=63474 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=2 budget=40960 expected_recv=65654 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[31833, 32856, 32702, 33096, 32368, 33095, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31833, 32856, 32702, 33096, 32368, 33095, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=40960 expected_recv=65425 expected_overflow=0 exact_recv_per_rank=[31833, 32856, 32702, 33096, 32368, 33095, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31833, 32856, 32702, 33096, 32368, 33095, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31833, 32856, 32702, 33096, 32368, 33095, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=40960 expected_recv=64966 expected_overflow=0 exact_recv_per_rank=[31833, 32856, 32702, 33096, 32368, 33095, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=2 budget=40960 expected_recv=65425 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=2 budget=40960 expected_recv=64966 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65504 expected_overflow=0 exact_recv_per_rank=[32738, 32962, 33096, 33096, 33096, 31871, 32493, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65803 expected_overflow=0 exact_recv_per_rank=[32738, 32962, 33096, 33096, 33096, 31871, 32493, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=40960 expected_recv=63768 expected_overflow=0 exact_recv_per_rank=[32738, 32962, 33096, 33096, 33096, 31871, 32493, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65504 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65803 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=40960 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[32738, 32962, 33096, 33096, 33096, 31871, 32493, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=2 budget=40960 expected_recv=63768 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[32738, 32962, 33096, 33096, 33096, 31871, 32493, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=2 budget=40960 expected_recv=66190 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65099 expected_overflow=0 exact_recv_per_rank=[32738, 32962, 33096, 33096, 33096, 31871, 32493, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[32738, 32962, 33096, 33096, 33096, 31871, 32493, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65099 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=2 budget=40960 expected_recv=65538 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32738, 32962, 33096, 33096, 33096, 31871, 32493, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32736, 33096, 32679, 32066, 33095, 32281, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65356 expected_overflow=0 exact_recv_per_rank=[33096, 32736, 33096, 32679, 32066, 33095, 32281, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=40960 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[33096, 32736, 33096, 32679, 32066, 33095, 32281, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65356 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65372 expected_overflow=0 exact_recv_per_rank=[33096, 32736, 33096, 32679, 32066, 33095, 32281, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=2 budget=40960 expected_recv=66190 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=40960 expected_recv=64994 expected_overflow=0 exact_recv_per_rank=[33096, 32736, 33096, 32679, 32066, 33095, 32281, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65372 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=40960 expected_recv=64828 expected_overflow=0 exact_recv_per_rank=[33096, 32736, 33096, 32679, 32066, 33095, 32281, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=2 budget=40960 expected_recv=64994 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32736, 33096, 32679, 32066, 33095, 32281, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=2 budget=40960 expected_recv=64828 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65164 expected_overflow=0 exact_recv_per_rank=[33096, 32736, 33096, 32679, 32066, 33095, 32281, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32539, 32768, 32693, 33096, 32074, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=2 budget=40960 expected_recv=65164 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=40960 expected_recv=64331 expected_overflow=0 exact_recv_per_rank=[33096, 32539, 32768, 32693, 33096, 32074, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=40960 expected_recv=64827 expected_overflow=0 exact_recv_per_rank=[33096, 32539, 32768, 32693, 33096, 32074, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=2 budget=40960 expected_recv=64331 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=2 budget=40960 expected_recv=64827 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65434 expected_overflow=0 exact_recv_per_rank=[33096, 32539, 32768, 32693, 33096, 32074, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65570 expected_overflow=0 exact_recv_per_rank=[33096, 32539, 32768, 32693, 33096, 32074, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[33096, 32539, 32768, 32693, 33096, 32074, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65434 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65570 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32539, 32768, 32693, 33096, 32074, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=2 budget=40960 expected_recv=65550 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32539, 32768, 32693, 33096, 32074, 32782, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65507 expected_overflow=0 exact_recv_per_rank=[32762, 32858, 33096, 32769, 32776, 32438, 33096, 32349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65615 expected_overflow=0 exact_recv_per_rank=[32762, 32858, 33096, 32769, 32776, 32438, 33096, 32349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=40960 expected_recv=64941 expected_overflow=0 exact_recv_per_rank=[32762, 32858, 33096, 32769, 32776, 32438, 33096, 32349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65507 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65615 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=2 budget=40960 expected_recv=64941 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65508 expected_overflow=0 exact_recv_per_rank=[32762, 32858, 33096, 32769, 32776, 32438, 33096, 32349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32762, 32858, 33096, 32769, 32776, 32438, 33096, 32349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32762, 32858, 33096, 32769, 32776, 32438, 33096, 32349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65508 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=40960 expected_recv=64785 expected_overflow=0 exact_recv_per_rank=[32762, 32858, 33096, 32769, 32776, 32438, 33096, 32349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65548 expected_overflow=0 exact_recv_per_rank=[32762, 32858, 33096, 32769, 32776, 32438, 33096, 32349] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=2 budget=40960 expected_recv=64785 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[32766, 31362, 32778, 32854, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=2 budget=40960 expected_recv=65548 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=40960 expected_recv=63007 expected_overflow=0 exact_recv_per_rank=[32766, 31362, 32778, 32854, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[32766, 31362, 32778, 32854, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65543 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=2 budget=40960 expected_recv=63007 actual_valid_recv=63872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=63872 slack_slots=18048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65574 expected_overflow=0 exact_recv_per_rank=[32766, 31362, 32778, 32854, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65676 expected_overflow=0 exact_recv_per_rank=[32766, 31362, 32778, 32854, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32766, 31362, 32778, 32854, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65574 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65676 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32766, 31362, 32778, 32854, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65910 expected_overflow=0 exact_recv_per_rank=[32766, 31362, 32778, 32854, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=40960 expected_recv=64624 expected_overflow=0 exact_recv_per_rank=[33096, 32282, 32766, 32578, 32780, 33096, 32778, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=2 budget=40960 expected_recv=65910 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32282, 32766, 32578, 32780, 33096, 32778, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65980 expected_overflow=0 exact_recv_per_rank=[33096, 32282, 32766, 32578, 32780, 33096, 32778, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=2 budget=40960 expected_recv=64624 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65980 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65507 expected_overflow=0 exact_recv_per_rank=[33096, 32282, 32766, 32578, 32780, 33096, 32778, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65370 expected_overflow=0 exact_recv_per_rank=[33096, 32282, 32766, 32578, 32780, 33096, 32778, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65521 expected_overflow=0 exact_recv_per_rank=[33096, 32282, 32766, 32578, 32780, 33096, 32778, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65507 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65370 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65514 expected_overflow=0 exact_recv_per_rank=[33096, 32282, 32766, 32578, 32780, 33096, 32778, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65521 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65514 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65580 expected_overflow=0 exact_recv_per_rank=[33096, 32282, 32766, 32578, 32780, 33096, 32778, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32183, 32769, 32843, 32296, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=2 budget=40960 expected_recv=65580 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=40960 expected_recv=64716 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32183, 32769, 32843, 32296, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65903 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32183, 32769, 32843, 32296, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=2 budget=40960 expected_recv=64716 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65903 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=40960 expected_recv=64610 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32183, 32769, 32843, 32296, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32183, 32769, 32843, 32296, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65497 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32183, 32769, 32843, 32296, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=2 budget=40960 expected_recv=64610 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65529 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32183, 32769, 32843, 32296, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65497 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65649 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32183, 32769, 32843, 32296, 32765, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=40960 expected_recv=64869 expected_overflow=0 exact_recv_per_rank=[32766, 32401, 33096, 33096, 32767, 32443, 32764, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=2 budget=40960 expected_recv=65649 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65160 expected_overflow=0 exact_recv_per_rank=[32766, 32401, 33096, 33096, 32767, 32443, 32764, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=2 budget=40960 expected_recv=64869 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[32766, 32401, 33096, 33096, 32767, 32443, 32764, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65160 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32766, 32401, 33096, 33096, 32767, 32443, 32764, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32766, 32401, 33096, 33096, 32767, 32443, 32764, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65543 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65559 expected_overflow=0 exact_recv_per_rank=[32766, 32401, 33096, 33096, 32767, 32443, 32764, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65641 expected_overflow=0 exact_recv_per_rank=[32766, 32401, 33096, 33096, 32767, 32443, 32764, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65559 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65641 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65132 expected_overflow=0 exact_recv_per_rank=[32766, 32401, 33096, 33096, 32767, 32443, 32764, 32811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=2 budget=40960 expected_recv=65132 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65194 expected_overflow=0 exact_recv_per_rank=[32785, 32395, 32828, 33096, 33096, 32435, 32413, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=40960 expected_recv=64904 expected_overflow=0 exact_recv_per_rank=[32785, 32395, 32828, 33096, 33096, 32435, 32413, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65194 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65605 expected_overflow=0 exact_recv_per_rank=[32785, 32395, 32828, 33096, 33096, 32435, 32413, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=2 budget=40960 expected_recv=64904 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65560 expected_overflow=0 exact_recv_per_rank=[32785, 32395, 32828, 33096, 33096, 32435, 32413, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32785, 32395, 32828, 33096, 33096, 32435, 32413, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65605 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=40960 expected_recv=64449 expected_overflow=0 exact_recv_per_rank=[32785, 32395, 32828, 33096, 33096, 32435, 32413, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32785, 32395, 32828, 33096, 33096, 32435, 32413, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=2 budget=40960 expected_recv=65560 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=2 budget=40960 expected_recv=64449 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32785, 32395, 32828, 33096, 33096, 32435, 32413, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=40960 expected_recv=64857 expected_overflow=0 exact_recv_per_rank=[32188, 32772, 32530, 33096, 32929, 32437, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=12 calls=2 budget=40960 expected_recv=64857 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=40960 expected_recv=65593 expected_overflow=0 exact_recv_per_rank=[32188, 32772, 32530, 33096, 32929, 32437, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=40960 expected_recv=64795 expected_overflow=0 exact_recv_per_rank=[32188, 32772, 32530, 33096, 32929, 32437, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=12 calls=2 budget=40960 expected_recv=65593 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=40960 expected_recv=64844 expected_overflow=0 exact_recv_per_rank=[32188, 32772, 32530, 33096, 32929, 32437, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=12 calls=2 budget=40960 expected_recv=64795 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32188, 32772, 32530, 33096, 32929, 32437, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32188, 32772, 32530, 33096, 32929, 32437, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32188, 32772, 32530, 33096, 32929, 32437, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=2 budget=40960 expected_recv=64844 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=40960 expected_recv=65623 expected_overflow=0 exact_recv_per_rank=[32188, 32772, 32530, 33096, 32929, 32437, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32539, 32769, 32341, 33096, 32438, 33096, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=12 calls=2 budget=40960 expected_recv=65623 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=40960 expected_recv=65595 expected_overflow=0 exact_recv_per_rank=[32539, 32769, 32341, 33096, 32438, 33096, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=2 budget=40960 expected_recv=65595 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=40960 expected_recv=64960 expected_overflow=0 exact_recv_per_rank=[32539, 32769, 32341, 33096, 32438, 33096, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=40960 expected_recv=64750 expected_overflow=0 exact_recv_per_rank=[32539, 32769, 32341, 33096, 32438, 33096, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=2 budget=40960 expected_recv=64960 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32539, 32769, 32341, 33096, 32438, 33096, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=40960 expected_recv=65494 expected_overflow=0 exact_recv_per_rank=[32539, 32769, 32341, 33096, 32438, 33096, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=13 calls=2 budget=40960 expected_recv=64750 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32539, 32769, 32341, 33096, 32438, 33096, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=40960 expected_recv=64913 expected_overflow=0 exact_recv_per_rank=[32539, 32769, 32341, 33096, 32438, 33096, 32769, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=13 calls=2 budget=40960 expected_recv=65494 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65412 expected_overflow=0 exact_recv_per_rank=[32445, 32766, 32443, 32762, 33096, 32766, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=2 budget=40960 expected_recv=64913 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65412 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[32445, 32766, 32443, 32762, 33096, 32766, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65534 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=40960 expected_recv=64877 expected_overflow=0 exact_recv_per_rank=[32445, 32766, 32443, 32762, 33096, 32766, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65002 expected_overflow=0 exact_recv_per_rank=[32445, 32766, 32443, 32762, 33096, 32766, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65515 expected_overflow=0 exact_recv_per_rank=[32445, 32766, 32443, 32762, 33096, 32766, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=2 budget=40960 expected_recv=64877 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65564 expected_overflow=0 exact_recv_per_rank=[32445, 32766, 32443, 32762, 33096, 32766, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65002 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32445, 32766, 32443, 32762, 33096, 32766, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65515 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32445, 32766, 32443, 32762, 33096, 32766, 32770, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=40960 expected_recv=64957 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32763, 32443, 32766, 32764, 32451, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=14 calls=2 budget=40960 expected_recv=65564 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32763, 32443, 32766, 32764, 32451, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=2 budget=40960 expected_recv=64957 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65596 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32763, 32443, 32766, 32764, 32451, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65596 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65464 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32763, 32443, 32766, 32764, 32451, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65021 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32763, 32443, 32766, 32764, 32451, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65464 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65358 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32763, 32443, 32766, 32764, 32451, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32763, 32443, 32766, 32764, 32451, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65508 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 32763, 32443, 32766, 32764, 32451, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65021 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[48804, 29473, 54821, 15591, 38933, 40995, 13334, 20193] avg=32768.0000 max=54821 max_rank=2 min=13334 min_rank=6 imbalance_ratio=1.673004 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65358 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=2 budget=40960 expected_recv=65508 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[48804, 29473, 54821, 15591, 38933, 40995, 13334, 20193] avg=32768.0000 max=54821 max_rank=2 min=13334 min_rank=6 imbalance_ratio=1.673004 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21725 max_load_reduction_pct=0.396290 moved_to_replicas=99310 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[48804, 29473, 54821, 15591, 38933, 40995, 13334, 20193] avg=32768.0000 max=54821 max_rank=2 min=13334 min_rank=6 imbalance_ratio=1.673004 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21725 max_load_reduction_pct=0.396290 moved_to_replicas=99310 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[48804, 29473, 54821, 15591, 38933, 40995, 13334, 20193] avg=32768.0000 max=54821 max_rank=2 min=13334 min_rank=6 imbalance_ratio=1.673004 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[48804, 29473, 54821, 15591, 38933, 40995, 13334, 20193] avg=32768.0000 max=54821 max_rank=2 min=13334 min_rank=6 imbalance_ratio=1.673004 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[48804, 29473, 54821, 15591, 38933, 40995, 13334, 20193] avg=32768.0000 max=54821 max_rank=2 min=13334 min_rank=6 imbalance_ratio=1.673004 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[48804, 29473, 54821, 15591, 38933, 40995, 13334, 20193] avg=32768.0000 max=54821 max_rank=2 min=13334 min_rank=6 imbalance_ratio=1.673004 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[48804, 29473, 54821, 15591, 38933, 40995, 13334, 20193] avg=32768.0000 max=54821 max_rank=2 min=13334 min_rank=6 imbalance_ratio=1.673004 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21725 max_load_reduction_pct=0.396290 moved_to_replicas=99310 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[35633, 65369, 16392, 32791, 32704, 44790, 10963, 23502] avg=32768.0000 max=65369 max_rank=1 min=10963 min_rank=6 imbalance_ratio=1.994904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21725 max_load_reduction_pct=0.396290 moved_to_replicas=99310 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[35633, 65369, 16392, 32791, 32704, 44790, 10963, 23502] avg=32768.0000 max=65369 max_rank=1 min=10963 min_rank=6 imbalance_ratio=1.994904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21725 max_load_reduction_pct=0.396290 moved_to_replicas=99310 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21725 max_load_reduction_pct=0.396290 moved_to_replicas=99310 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21725 max_load_reduction_pct=0.396290 moved_to_replicas=99310 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[33096, 30472, 33096, 33096, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30472 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=21725 max_load_reduction_pct=0.396290 moved_to_replicas=99310 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 31170, 32791, 32704, 33097, 33095, 33095] avg=32768.0000 max=33097 max_rank=5 min=31170 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32272 max_load_reduction_pct=0.493690 moved_to_replicas=93734 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 31170, 32791, 32704, 33097, 33095, 33095] avg=32768.0000 max=33097 max_rank=5 min=31170 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32272 max_load_reduction_pct=0.493690 moved_to_replicas=93734 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[35633, 65369, 16392, 32791, 32704, 44790, 10963, 23502] avg=32768.0000 max=65369 max_rank=1 min=10963 min_rank=6 imbalance_ratio=1.994904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[35633, 65369, 16392, 32791, 32704, 44790, 10963, 23502] avg=32768.0000 max=65369 max_rank=1 min=10963 min_rank=6 imbalance_ratio=1.994904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[35633, 65369, 16392, 32791, 32704, 44790, 10963, 23502] avg=32768.0000 max=65369 max_rank=1 min=10963 min_rank=6 imbalance_ratio=1.994904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[35633, 65369, 16392, 32791, 32704, 44790, 10963, 23502] avg=32768.0000 max=65369 max_rank=1 min=10963 min_rank=6 imbalance_ratio=1.994904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[35633, 65369, 16392, 32791, 32704, 44790, 10963, 23502] avg=32768.0000 max=65369 max_rank=1 min=10963 min_rank=6 imbalance_ratio=1.994904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[52, 32856, 32702, 58569, 32368, 370, 61204, 44023] avg=32768.0000 max=61204 max_rank=6 min=52 min_rank=0 imbalance_ratio=1.867798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[35633, 65369, 16392, 32791, 32704, 44790, 10963, 23502] avg=32768.0000 max=65369 max_rank=1 min=10963 min_rank=6 imbalance_ratio=1.994904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 31170, 32791, 32704, 33097, 33095, 33095] avg=32768.0000 max=33097 max_rank=5 min=31170 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32272 max_load_reduction_pct=0.493690 moved_to_replicas=93734 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 31170, 32791, 32704, 33097, 33095, 33095] avg=32768.0000 max=33097 max_rank=5 min=31170 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32272 max_load_reduction_pct=0.493690 moved_to_replicas=93734 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[52, 32856, 32702, 58569, 32368, 370, 61204, 44023] avg=32768.0000 max=61204 max_rank=6 min=52 min_rank=0 imbalance_ratio=1.867798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 31170, 32791, 32704, 33097, 33095, 33095] avg=32768.0000 max=33097 max_rank=5 min=31170 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32272 max_load_reduction_pct=0.493690 moved_to_replicas=93734 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 31170, 32791, 32704, 33097, 33095, 33095] avg=32768.0000 max=33097 max_rank=5 min=31170 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32272 max_load_reduction_pct=0.493690 moved_to_replicas=93734 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 31170, 32791, 32704, 33097, 33095, 33095] avg=32768.0000 max=33097 max_rank=5 min=31170 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32272 max_load_reduction_pct=0.493690 moved_to_replicas=93734 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[31833, 32856, 32702, 33096, 32368, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31833 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=28107 max_load_reduction_pct=0.459235 moved_to_replicas=129022 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[33096, 33096, 31170, 32791, 32704, 33097, 33095, 33095] avg=32768.0000 max=33097 max_rank=5 min=31170 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=32272 max_load_reduction_pct=0.493690 moved_to_replicas=93734 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[31833, 32856, 32702, 33096, 32368, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31833 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=28107 max_load_reduction_pct=0.459235 moved_to_replicas=129022 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[52, 32856, 32702, 58569, 32368, 370, 61204, 44023] avg=32768.0000 max=61204 max_rank=6 min=52 min_rank=0 imbalance_ratio=1.867798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[52, 32856, 32702, 58569, 32368, 370, 61204, 44023] avg=32768.0000 max=61204 max_rank=6 min=52 min_rank=0 imbalance_ratio=1.867798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[52, 32856, 32702, 58569, 32368, 370, 61204, 44023] avg=32768.0000 max=61204 max_rank=6 min=52 min_rank=0 imbalance_ratio=1.867798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[32738, 32962, 4672, 62542, 65496, 30849, 93, 32792] avg=32768.0000 max=65496 max_rank=4 min=93 min_rank=6 imbalance_ratio=1.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[52, 32856, 32702, 58569, 32368, 370, 61204, 44023] avg=32768.0000 max=61204 max_rank=6 min=52 min_rank=0 imbalance_ratio=1.867798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[52, 32856, 32702, 58569, 32368, 370, 61204, 44023] avg=32768.0000 max=61204 max_rank=6 min=52 min_rank=0 imbalance_ratio=1.867798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[31833, 32856, 32702, 33096, 32368, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31833 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=28107 max_load_reduction_pct=0.459235 moved_to_replicas=129022 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[52, 32856, 32702, 58569, 32368, 370, 61204, 44023] avg=32768.0000 max=61204 max_rank=6 min=52 min_rank=0 imbalance_ratio=1.867798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[31833, 32856, 32702, 33096, 32368, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31833 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=28107 max_load_reduction_pct=0.459235 moved_to_replicas=129022 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[32738, 32962, 4672, 62542, 65496, 30849, 93, 32792] avg=32768.0000 max=65496 max_rank=4 min=93 min_rank=6 imbalance_ratio=1.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[31833, 32856, 32702, 33096, 32368, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31833 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=28107 max_load_reduction_pct=0.459235 moved_to_replicas=129022 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[32738, 32962, 33096, 33096, 33096, 31871, 32493, 32792] avg=32768.0000 max=33096 max_rank=2 min=31871 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32400 max_load_reduction_pct=0.494687 moved_to_replicas=121935 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[31833, 32856, 32702, 33096, 32368, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31833 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=28107 max_load_reduction_pct=0.459235 moved_to_replicas=129022 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[31833, 32856, 32702, 33096, 32368, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31833 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=28107 max_load_reduction_pct=0.459235 moved_to_replicas=129022 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[31833, 32856, 32702, 33096, 32368, 33095, 33097, 33097] avg=32768.0000 max=33097 max_rank=6 min=31833 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=28107 max_load_reduction_pct=0.459235 moved_to_replicas=129022 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[32738, 32962, 33096, 33096, 33096, 31871, 32493, 32792] avg=32768.0000 max=33096 max_rank=2 min=31871 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32400 max_load_reduction_pct=0.494687 moved_to_replicas=121935 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[32738, 32962, 4672, 62542, 65496, 30849, 93, 32792] avg=32768.0000 max=65496 max_rank=4 min=93 min_rank=6 imbalance_ratio=1.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[32738, 32962, 4672, 62542, 65496, 30849, 93, 32792] avg=32768.0000 max=65496 max_rank=4 min=93 min_rank=6 imbalance_ratio=1.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[32738, 32962, 4672, 62542, 65496, 30849, 93, 32792] avg=32768.0000 max=65496 max_rank=4 min=93 min_rank=6 imbalance_ratio=1.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[65141, 32736, 23987, 32679, 581, 56914, 236, 49870] avg=32768.0000 max=65141 max_rank=0 min=236 min_rank=6 imbalance_ratio=1.987946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[32738, 32962, 4672, 62542, 65496, 30849, 93, 32792] avg=32768.0000 max=65496 max_rank=4 min=93 min_rank=6 imbalance_ratio=1.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[32738, 32962, 4672, 62542, 65496, 30849, 93, 32792] avg=32768.0000 max=65496 max_rank=4 min=93 min_rank=6 imbalance_ratio=1.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[32738, 32962, 4672, 62542, 65496, 30849, 93, 32792] avg=32768.0000 max=65496 max_rank=4 min=93 min_rank=6 imbalance_ratio=1.998779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[32738, 32962, 33096, 33096, 33096, 31871, 32493, 32792] avg=32768.0000 max=33096 max_rank=2 min=31871 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32400 max_load_reduction_pct=0.494687 moved_to_replicas=121935 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[32738, 32962, 33096, 33096, 33096, 31871, 32493, 32792] avg=32768.0000 max=33096 max_rank=2 min=31871 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32400 max_load_reduction_pct=0.494687 moved_to_replicas=121935 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[65141, 32736, 23987, 32679, 581, 56914, 236, 49870] avg=32768.0000 max=65141 max_rank=0 min=236 min_rank=6 imbalance_ratio=1.987946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[32738, 32962, 33096, 33096, 33096, 31871, 32493, 32792] avg=32768.0000 max=33096 max_rank=2 min=31871 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32400 max_load_reduction_pct=0.494687 moved_to_replicas=121935 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32736, 33096, 32679, 32066, 33095, 32281, 33095] avg=32768.0000 max=33096 max_rank=0 min=32066 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32045 max_load_reduction_pct=0.491933 moved_to_replicas=142766 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[32738, 32962, 33096, 33096, 33096, 31871, 32493, 32792] avg=32768.0000 max=33096 max_rank=2 min=31871 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32400 max_load_reduction_pct=0.494687 moved_to_replicas=121935 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[32738, 32962, 33096, 33096, 33096, 31871, 32493, 32792] avg=32768.0000 max=33096 max_rank=2 min=31871 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32400 max_load_reduction_pct=0.494687 moved_to_replicas=121935 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[32738, 32962, 33096, 33096, 33096, 31871, 32493, 32792] avg=32768.0000 max=33096 max_rank=2 min=31871 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32400 max_load_reduction_pct=0.494687 moved_to_replicas=121935 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32736, 33096, 32679, 32066, 33095, 32281, 33095] avg=32768.0000 max=33096 max_rank=0 min=32066 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32045 max_load_reduction_pct=0.491933 moved_to_replicas=142766 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[65141, 32736, 23987, 32679, 581, 56914, 236, 49870] avg=32768.0000 max=65141 max_rank=0 min=236 min_rank=6 imbalance_ratio=1.987946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[65141, 32736, 23987, 32679, 581, 56914, 236, 49870] avg=32768.0000 max=65141 max_rank=0 min=236 min_rank=6 imbalance_ratio=1.987946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[65141, 32736, 23987, 32679, 581, 56914, 236, 49870] avg=32768.0000 max=65141 max_rank=0 min=236 min_rank=6 imbalance_ratio=1.987946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[33375, 3, 32768, 255, 97237, 190, 32782, 65534] avg=32768.0000 max=97237 max_rank=4 min=3 min_rank=1 imbalance_ratio=2.967438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[65141, 32736, 23987, 32679, 581, 56914, 236, 49870] avg=32768.0000 max=65141 max_rank=0 min=236 min_rank=6 imbalance_ratio=1.987946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[65141, 32736, 23987, 32679, 581, 56914, 236, 49870] avg=32768.0000 max=65141 max_rank=0 min=236 min_rank=6 imbalance_ratio=1.987946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32736, 33096, 32679, 32066, 33095, 32281, 33095] avg=32768.0000 max=33096 max_rank=0 min=32066 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32045 max_load_reduction_pct=0.491933 moved_to_replicas=142766 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[65141, 32736, 23987, 32679, 581, 56914, 236, 49870] avg=32768.0000 max=65141 max_rank=0 min=236 min_rank=6 imbalance_ratio=1.987946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[33375, 3, 32768, 255, 97237, 190, 32782, 65534] avg=32768.0000 max=97237 max_rank=4 min=3 min_rank=1 imbalance_ratio=2.967438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32736, 33096, 32679, 32066, 33095, 32281, 33095] avg=32768.0000 max=33096 max_rank=0 min=32066 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32045 max_load_reduction_pct=0.491933 moved_to_replicas=142766 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32736, 33096, 32679, 32066, 33095, 32281, 33095] avg=32768.0000 max=33096 max_rank=0 min=32066 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32045 max_load_reduction_pct=0.491933 moved_to_replicas=142766 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[33096, 32539, 32768, 32693, 33096, 32074, 32782, 33096] avg=32768.0000 max=33096 max_rank=0 min=32074 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64141 max_load_reduction_pct=0.659636 moved_to_replicas=193713 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32736, 33096, 32679, 32066, 33095, 32281, 33095] avg=32768.0000 max=33096 max_rank=0 min=32066 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32045 max_load_reduction_pct=0.491933 moved_to_replicas=142766 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32736, 33096, 32679, 32066, 33095, 32281, 33095] avg=32768.0000 max=33096 max_rank=0 min=32066 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32045 max_load_reduction_pct=0.491933 moved_to_replicas=142766 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32736, 33096, 32679, 32066, 33095, 32281, 33095] avg=32768.0000 max=33096 max_rank=0 min=32066 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32045 max_load_reduction_pct=0.491933 moved_to_replicas=142766 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[33096, 32539, 32768, 32693, 33096, 32074, 32782, 33096] avg=32768.0000 max=33096 max_rank=0 min=32074 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64141 max_load_reduction_pct=0.659636 moved_to_replicas=193713 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[33375, 3, 32768, 255, 97237, 190, 32782, 65534] avg=32768.0000 max=97237 max_rank=4 min=3 min_rank=1 imbalance_ratio=2.967438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[33375, 3, 32768, 255, 97237, 190, 32782, 65534] avg=32768.0000 max=97237 max_rank=4 min=3 min_rank=1 imbalance_ratio=2.967438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 36069, 32769, 32776, 5, 65529, 29376] avg=32768.0000 max=65529 max_rank=6 min=5 min_rank=5 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[33375, 3, 32768, 255, 97237, 190, 32782, 65534] avg=32768.0000 max=97237 max_rank=4 min=3 min_rank=1 imbalance_ratio=2.967438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[33375, 3, 32768, 255, 97237, 190, 32782, 65534] avg=32768.0000 max=97237 max_rank=4 min=3 min_rank=1 imbalance_ratio=2.967438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[33375, 3, 32768, 255, 97237, 190, 32782, 65534] avg=32768.0000 max=97237 max_rank=4 min=3 min_rank=1 imbalance_ratio=2.967438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[33096, 32539, 32768, 32693, 33096, 32074, 32782, 33096] avg=32768.0000 max=33096 max_rank=0 min=32074 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64141 max_load_reduction_pct=0.659636 moved_to_replicas=193713 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[33375, 3, 32768, 255, 97237, 190, 32782, 65534] avg=32768.0000 max=97237 max_rank=4 min=3 min_rank=1 imbalance_ratio=2.967438 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 36069, 32769, 32776, 5, 65529, 29376] avg=32768.0000 max=65529 max_rank=6 min=5 min_rank=5 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[33096, 32539, 32768, 32693, 33096, 32074, 32782, 33096] avg=32768.0000 max=33096 max_rank=0 min=32074 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64141 max_load_reduction_pct=0.659636 moved_to_replicas=193713 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 33096, 32769, 32776, 32438, 33096, 32349] avg=32768.0000 max=33096 max_rank=2 min=32349 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=70405 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[33096, 32539, 32768, 32693, 33096, 32074, 32782, 33096] avg=32768.0000 max=33096 max_rank=0 min=32074 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64141 max_load_reduction_pct=0.659636 moved_to_replicas=193713 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[33096, 32539, 32768, 32693, 33096, 32074, 32782, 33096] avg=32768.0000 max=33096 max_rank=0 min=32074 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64141 max_load_reduction_pct=0.659636 moved_to_replicas=193713 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[33096, 32539, 32768, 32693, 33096, 32074, 32782, 33096] avg=32768.0000 max=33096 max_rank=0 min=32074 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64141 max_load_reduction_pct=0.659636 moved_to_replicas=193713 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[33096, 32539, 32768, 32693, 33096, 32074, 32782, 33096] avg=32768.0000 max=33096 max_rank=0 min=32074 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=64141 max_load_reduction_pct=0.659636 moved_to_replicas=193713 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 33096, 32769, 32776, 32438, 33096, 32349] avg=32768.0000 max=33096 max_rank=2 min=32349 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=70405 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 36069, 32769, 32776, 5, 65529, 29376] avg=32768.0000 max=65529 max_rank=6 min=5 min_rank=5 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 36069, 32769, 32776, 5, 65529, 29376] avg=32768.0000 max=65529 max_rank=6 min=5 min_rank=5 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[32766, 61, 12, 32854, 33638, 94571, 2763, 65479] avg=32768.0000 max=94571 max_rank=5 min=12 min_rank=2 imbalance_ratio=2.886078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 36069, 32769, 32776, 5, 65529, 29376] avg=32768.0000 max=65529 max_rank=6 min=5 min_rank=5 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 36069, 32769, 32776, 5, 65529, 29376] avg=32768.0000 max=65529 max_rank=6 min=5 min_rank=5 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 33096, 32769, 32776, 32438, 33096, 32349] avg=32768.0000 max=33096 max_rank=2 min=32349 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=70405 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 36069, 32769, 32776, 5, 65529, 29376] avg=32768.0000 max=65529 max_rank=6 min=5 min_rank=5 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 36069, 32769, 32776, 5, 65529, 29376] avg=32768.0000 max=65529 max_rank=6 min=5 min_rank=5 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[32766, 61, 12, 32854, 33638, 94571, 2763, 65479] avg=32768.0000 max=94571 max_rank=5 min=12 min_rank=2 imbalance_ratio=2.886078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[32766, 31362, 32778, 32854, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61475 max_load_reduction_pct=0.650041 moved_to_replicas=187346 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 33096, 32769, 32776, 32438, 33096, 32349] avg=32768.0000 max=33096 max_rank=2 min=32349 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=70405 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 33096, 32769, 32776, 32438, 33096, 32349] avg=32768.0000 max=33096 max_rank=2 min=32349 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=70405 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 33096, 32769, 32776, 32438, 33096, 32349] avg=32768.0000 max=33096 max_rank=2 min=32349 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=70405 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 33096, 32769, 32776, 32438, 33096, 32349] avg=32768.0000 max=33096 max_rank=2 min=32349 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=70405 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[32762, 32858, 33096, 32769, 32776, 32438, 33096, 32349] avg=32768.0000 max=33096 max_rank=2 min=32349 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=70405 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[32766, 31362, 32778, 32854, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61475 max_load_reduction_pct=0.650041 moved_to_replicas=187346 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[32766, 61, 12, 32854, 33638, 94571, 2763, 65479] avg=32768.0000 max=94571 max_rank=5 min=12 min_rank=2 imbalance_ratio=2.886078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[34661, 3, 32766, 31013, 32780, 65375, 32778, 32768] avg=32768.0000 max=65375 max_rank=5 min=3 min_rank=1 imbalance_ratio=1.995087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[32766, 61, 12, 32854, 33638, 94571, 2763, 65479] avg=32768.0000 max=94571 max_rank=5 min=12 min_rank=2 imbalance_ratio=2.886078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[32766, 61, 12, 32854, 33638, 94571, 2763, 65479] avg=32768.0000 max=94571 max_rank=5 min=12 min_rank=2 imbalance_ratio=2.886078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[32766, 61, 12, 32854, 33638, 94571, 2763, 65479] avg=32768.0000 max=94571 max_rank=5 min=12 min_rank=2 imbalance_ratio=2.886078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[32766, 31362, 32778, 32854, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61475 max_load_reduction_pct=0.650041 moved_to_replicas=187346 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[32766, 61, 12, 32854, 33638, 94571, 2763, 65479] avg=32768.0000 max=94571 max_rank=5 min=12 min_rank=2 imbalance_ratio=2.886078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[34661, 3, 32766, 31013, 32780, 65375, 32778, 32768] avg=32768.0000 max=65375 max_rank=5 min=3 min_rank=1 imbalance_ratio=1.995087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[32766, 61, 12, 32854, 33638, 94571, 2763, 65479] avg=32768.0000 max=94571 max_rank=5 min=12 min_rank=2 imbalance_ratio=2.886078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32282, 32766, 32578, 32780, 33096, 32778, 32768] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32279 max_load_reduction_pct=0.493751 moved_to_replicas=66069 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[32766, 31362, 32778, 32854, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61475 max_load_reduction_pct=0.650041 moved_to_replicas=187346 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[32766, 31362, 32778, 32854, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61475 max_load_reduction_pct=0.650041 moved_to_replicas=187346 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[32766, 31362, 32778, 32854, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61475 max_load_reduction_pct=0.650041 moved_to_replicas=187346 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[32766, 31362, 32778, 32854, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61475 max_load_reduction_pct=0.650041 moved_to_replicas=187346 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32282, 32766, 32578, 32780, 33096, 32778, 32768] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32279 max_load_reduction_pct=0.493751 moved_to_replicas=66069 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[32766, 31362, 32778, 32854, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31362 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61475 max_load_reduction_pct=0.650041 moved_to_replicas=187346 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[34661, 3, 32766, 31013, 32780, 65375, 32778, 32768] avg=32768.0000 max=65375 max_rank=5 min=3 min_rank=1 imbalance_ratio=1.995087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[33631, 64854, 2, 32769, 32843, 3, 32765, 65277] avg=32768.0000 max=65277 max_rank=7 min=2 min_rank=2 imbalance_ratio=1.992096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[34661, 3, 32766, 31013, 32780, 65375, 32778, 32768] avg=32768.0000 max=65375 max_rank=5 min=3 min_rank=1 imbalance_ratio=1.995087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[34661, 3, 32766, 31013, 32780, 65375, 32778, 32768] avg=32768.0000 max=65375 max_rank=5 min=3 min_rank=1 imbalance_ratio=1.995087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[34661, 3, 32766, 31013, 32780, 65375, 32778, 32768] avg=32768.0000 max=65375 max_rank=5 min=3 min_rank=1 imbalance_ratio=1.995087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32282, 32766, 32578, 32780, 33096, 32778, 32768] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32279 max_load_reduction_pct=0.493751 moved_to_replicas=66069 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[34661, 3, 32766, 31013, 32780, 65375, 32778, 32768] avg=32768.0000 max=65375 max_rank=5 min=3 min_rank=1 imbalance_ratio=1.995087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[33631, 64854, 2, 32769, 32843, 3, 32765, 65277] avg=32768.0000 max=65277 max_rank=7 min=2 min_rank=2 imbalance_ratio=1.992096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[33096, 33096, 32183, 32769, 32843, 32296, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32183 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.492991 moved_to_replicas=129174 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[34661, 3, 32766, 31013, 32780, 65375, 32778, 32768] avg=32768.0000 max=65375 max_rank=5 min=3 min_rank=1 imbalance_ratio=1.995087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32282, 32766, 32578, 32780, 33096, 32778, 32768] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32279 max_load_reduction_pct=0.493751 moved_to_replicas=66069 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32282, 32766, 32578, 32780, 33096, 32778, 32768] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32279 max_load_reduction_pct=0.493751 moved_to_replicas=66069 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32282, 32766, 32578, 32780, 33096, 32778, 32768] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32279 max_load_reduction_pct=0.493751 moved_to_replicas=66069 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32282, 32766, 32578, 32780, 33096, 32778, 32768] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32279 max_load_reduction_pct=0.493751 moved_to_replicas=66069 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[33096, 33096, 32183, 32769, 32843, 32296, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32183 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.492991 moved_to_replicas=129174 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[33096, 32282, 32766, 32578, 32780, 33096, 32778, 32768] avg=32768.0000 max=33096 max_rank=0 min=32282 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32279 max_load_reduction_pct=0.493751 moved_to_replicas=66069 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[33631, 64854, 2, 32769, 32843, 3, 32765, 65277] avg=32768.0000 max=65277 max_rank=7 min=2 min_rank=2 imbalance_ratio=1.992096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[32766, 3, 98260, 65536, 1, 3, 32764, 32811] avg=32768.0000 max=98260 max_rank=2 min=1 min_rank=4 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[33631, 64854, 2, 32769, 32843, 3, 32765, 65277] avg=32768.0000 max=65277 max_rank=7 min=2 min_rank=2 imbalance_ratio=1.992096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[33631, 64854, 2, 32769, 32843, 3, 32765, 65277] avg=32768.0000 max=65277 max_rank=7 min=2 min_rank=2 imbalance_ratio=1.992096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[33631, 64854, 2, 32769, 32843, 3, 32765, 65277] avg=32768.0000 max=65277 max_rank=7 min=2 min_rank=2 imbalance_ratio=1.992096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[33096, 33096, 32183, 32769, 32843, 32296, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32183 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.492991 moved_to_replicas=129174 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[33631, 64854, 2, 32769, 32843, 3, 32765, 65277] avg=32768.0000 max=65277 max_rank=7 min=2 min_rank=2 imbalance_ratio=1.992096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[32766, 32401, 33096, 33096, 32767, 32443, 32764, 32811] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=195012 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[32766, 3, 98260, 65536, 1, 3, 32764, 32811] avg=32768.0000 max=98260 max_rank=2 min=1 min_rank=4 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[33631, 64854, 2, 32769, 32843, 3, 32765, 65277] avg=32768.0000 max=65277 max_rank=7 min=2 min_rank=2 imbalance_ratio=1.992096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[33096, 33096, 32183, 32769, 32843, 32296, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32183 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.492991 moved_to_replicas=129174 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[33096, 33096, 32183, 32769, 32843, 32296, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32183 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.492991 moved_to_replicas=129174 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[33096, 33096, 32183, 32769, 32843, 32296, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32183 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.492991 moved_to_replicas=129174 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[33096, 33096, 32183, 32769, 32843, 32296, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32183 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.492991 moved_to_replicas=129174 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[32766, 32401, 33096, 33096, 32767, 32443, 32764, 32811] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=195012 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[32766, 3, 98260, 65536, 1, 3, 32764, 32811] avg=32768.0000 max=98260 max_rank=2 min=1 min_rank=4 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[33096, 33096, 32183, 32769, 32843, 32296, 32765, 33096] avg=32768.0000 max=33096 max_rank=0 min=32183 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32181 max_load_reduction_pct=0.492991 moved_to_replicas=129174 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[32785, 55, 32828, 65528, 65436, 3, 6, 65503] avg=32768.0000 max=65528 max_rank=3 min=3 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[32766, 3, 98260, 65536, 1, 3, 32764, 32811] avg=32768.0000 max=98260 max_rank=2 min=1 min_rank=4 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[32766, 32401, 33096, 33096, 32767, 32443, 32764, 32811] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=195012 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[32766, 3, 98260, 65536, 1, 3, 32764, 32811] avg=32768.0000 max=98260 max_rank=2 min=1 min_rank=4 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[32785, 32395, 32828, 33096, 33096, 32435, 32413, 33096] avg=32768.0000 max=33096 max_rank=3 min=32395 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194210 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[32766, 3, 98260, 65536, 1, 3, 32764, 32811] avg=32768.0000 max=98260 max_rank=2 min=1 min_rank=4 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[32766, 3, 98260, 65536, 1, 3, 32764, 32811] avg=32768.0000 max=98260 max_rank=2 min=1 min_rank=4 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[32785, 55, 32828, 65528, 65436, 3, 6, 65503] avg=32768.0000 max=65528 max_rank=3 min=3 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[32766, 3, 98260, 65536, 1, 3, 32764, 32811] avg=32768.0000 max=98260 max_rank=2 min=1 min_rank=4 imbalance_ratio=2.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[32766, 32401, 33096, 33096, 32767, 32443, 32764, 32811] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=195012 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[32766, 32401, 33096, 33096, 32767, 32443, 32764, 32811] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=195012 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[32766, 32401, 33096, 33096, 32767, 32443, 32764, 32811] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=195012 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[32766, 32401, 33096, 33096, 32767, 32443, 32764, 32811] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=195012 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[32785, 32395, 32828, 33096, 33096, 32435, 32413, 33096] avg=32768.0000 max=33096 max_rank=3 min=32395 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194210 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[32785, 55, 32828, 65528, 65436, 3, 6, 65503] avg=32768.0000 max=65528 max_rank=3 min=3 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[31500, 32772, 109, 65532, 32929, 1, 65517, 33784] avg=32768.0000 max=65532 max_rank=3 min=1 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[32766, 32401, 33096, 33096, 32767, 32443, 32764, 32811] avg=32768.0000 max=33096 max_rank=2 min=32401 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65164 max_load_reduction_pct=0.663179 moved_to_replicas=195012 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[32785, 55, 32828, 65528, 65436, 3, 6, 65503] avg=32768.0000 max=65528 max_rank=3 min=3 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[32785, 32395, 32828, 33096, 33096, 32435, 32413, 33096] avg=32768.0000 max=33096 max_rank=3 min=32395 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194210 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[32188, 32772, 32530, 33096, 32929, 32437, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32188 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=132289 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[32785, 55, 32828, 65528, 65436, 3, 6, 65503] avg=32768.0000 max=65528 max_rank=3 min=3 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[32785, 55, 32828, 65528, 65436, 3, 6, 65503] avg=32768.0000 max=65528 max_rank=3 min=3 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[31500, 32772, 109, 65532, 32929, 1, 65517, 33784] avg=32768.0000 max=65532 max_rank=3 min=1 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[32785, 55, 32828, 65528, 65436, 3, 6, 65503] avg=32768.0000 max=65528 max_rank=3 min=3 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[32785, 55, 32828, 65528, 65436, 3, 6, 65503] avg=32768.0000 max=65528 max_rank=3 min=3 min_rank=5 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[32785, 32395, 32828, 33096, 33096, 32435, 32413, 33096] avg=32768.0000 max=33096 max_rank=3 min=32395 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194210 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[32785, 32395, 32828, 33096, 33096, 32435, 32413, 33096] avg=32768.0000 max=33096 max_rank=3 min=32395 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194210 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[32785, 32395, 32828, 33096, 33096, 32435, 32413, 33096] avg=32768.0000 max=33096 max_rank=3 min=32395 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194210 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[32188, 32772, 32530, 33096, 32929, 32437, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32188 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=132289 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[32785, 32395, 32828, 33096, 33096, 32435, 32413, 33096] avg=32768.0000 max=33096 max_rank=3 min=32395 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194210 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[31500, 32772, 109, 65532, 32929, 1, 65517, 33784] avg=32768.0000 max=65532 max_rank=3 min=1 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[0, 32769, 6, 65431, 4, 65530, 32769, 65635] avg=32768.0000 max=65635 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[32785, 32395, 32828, 33096, 33096, 32435, 32413, 33096] avg=32768.0000 max=33096 max_rank=3 min=32395 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194210 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[31500, 32772, 109, 65532, 32929, 1, 65517, 33784] avg=32768.0000 max=65532 max_rank=3 min=1 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[32188, 32772, 32530, 33096, 32929, 32437, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32188 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=132289 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[31500, 32772, 109, 65532, 32929, 1, 65517, 33784] avg=32768.0000 max=65532 max_rank=3 min=1 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[32539, 32769, 32341, 33096, 32438, 33096, 32769, 33096] avg=32768.0000 max=33096 max_rank=3 min=32341 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32539 max_load_reduction_pct=0.495757 moved_to_replicas=194255 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[0, 32769, 6, 65431, 4, 65530, 32769, 65635] avg=32768.0000 max=65635 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[31500, 32772, 109, 65532, 32929, 1, 65517, 33784] avg=32768.0000 max=65532 max_rank=3 min=1 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[31500, 32772, 109, 65532, 32929, 1, 65517, 33784] avg=32768.0000 max=65532 max_rank=3 min=1 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[31500, 32772, 109, 65532, 32929, 1, 65517, 33784] avg=32768.0000 max=65532 max_rank=3 min=1 min_rank=5 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[32188, 32772, 32530, 33096, 32929, 32437, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32188 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=132289 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[32188, 32772, 32530, 33096, 32929, 32437, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32188 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=132289 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[32539, 32769, 32341, 33096, 32438, 33096, 32769, 33096] avg=32768.0000 max=33096 max_rank=3 min=32341 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32539 max_load_reduction_pct=0.495757 moved_to_replicas=194255 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[32188, 32772, 32530, 33096, 32929, 32437, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32188 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=132289 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[32188, 32772, 32530, 33096, 32929, 32437, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32188 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=132289 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[0, 32769, 6, 65431, 4, 65530, 32769, 65635] avg=32768.0000 max=65635 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[2788, 32766, 6, 32762, 98299, 32766, 4, 62753] avg=32768.0000 max=98299 max_rank=4 min=4 min_rank=6 imbalance_ratio=2.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[32188, 32772, 32530, 33096, 32929, 32437, 33096, 33096] avg=32768.0000 max=33096 max_rank=3 min=32188 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32436 max_load_reduction_pct=0.494964 moved_to_replicas=132289 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[0, 32769, 6, 65431, 4, 65530, 32769, 65635] avg=32768.0000 max=65635 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[2788, 32766, 6, 32762, 98299, 32766, 4, 62753] avg=32768.0000 max=98299 max_rank=4 min=4 min_rank=6 imbalance_ratio=2.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[0, 32769, 6, 65431, 4, 65530, 32769, 65635] avg=32768.0000 max=65635 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[32539, 32769, 32341, 33096, 32438, 33096, 32769, 33096] avg=32768.0000 max=33096 max_rank=3 min=32341 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32539 max_load_reduction_pct=0.495757 moved_to_replicas=194255 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[32445, 32766, 32443, 32762, 33096, 32766, 32770, 33096] avg=32768.0000 max=33096 max_rank=4 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65203 max_load_reduction_pct=0.663313 moved_to_replicas=189289 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[0, 32769, 6, 65431, 4, 65530, 32769, 65635] avg=32768.0000 max=65635 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[32539, 32769, 32341, 33096, 32438, 33096, 32769, 33096] avg=32768.0000 max=33096 max_rank=3 min=32341 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32539 max_load_reduction_pct=0.495757 moved_to_replicas=194255 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[0, 32769, 6, 65431, 4, 65530, 32769, 65635] avg=32768.0000 max=65635 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[0, 32769, 6, 65431, 4, 65530, 32769, 65635] avg=32768.0000 max=65635 max_rank=7 min=0 min_rank=0 imbalance_ratio=2.003021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[32445, 32766, 32443, 32762, 33096, 32766, 32770, 33096] avg=32768.0000 max=33096 max_rank=4 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65203 max_load_reduction_pct=0.663313 moved_to_replicas=189289 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[32539, 32769, 32341, 33096, 32438, 33096, 32769, 33096] avg=32768.0000 max=33096 max_rank=3 min=32341 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32539 max_load_reduction_pct=0.495757 moved_to_replicas=194255 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[32539, 32769, 32341, 33096, 32438, 33096, 32769, 33096] avg=32768.0000 max=33096 max_rank=3 min=32341 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32539 max_load_reduction_pct=0.495757 moved_to_replicas=194255 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[32539, 32769, 32341, 33096, 32438, 33096, 32769, 33096] avg=32768.0000 max=33096 max_rank=3 min=32341 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32539 max_load_reduction_pct=0.495757 moved_to_replicas=194255 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[32539, 32769, 32341, 33096, 32438, 33096, 32769, 33096] avg=32768.0000 max=33096 max_rank=3 min=32341 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32539 max_load_reduction_pct=0.495757 moved_to_replicas=194255 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[32765, 65536, 32763, 3, 32766, 32764, 28, 65519] avg=32768.0000 max=65536 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[2788, 32766, 6, 32762, 98299, 32766, 4, 62753] avg=32768.0000 max=98299 max_rank=4 min=4 min_rank=6 imbalance_ratio=2.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[2788, 32766, 6, 32762, 98299, 32766, 4, 62753] avg=32768.0000 max=98299 max_rank=4 min=4 min_rank=6 imbalance_ratio=2.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[32765, 65536, 32763, 3, 32766, 32764, 28, 65519] avg=32768.0000 max=65536 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[32765, 33096, 32763, 32443, 32766, 32764, 32451, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=129573 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[2788, 32766, 6, 32762, 98299, 32766, 4, 62753] avg=32768.0000 max=98299 max_rank=4 min=4 min_rank=6 imbalance_ratio=2.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[32445, 32766, 32443, 32762, 33096, 32766, 32770, 33096] avg=32768.0000 max=33096 max_rank=4 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65203 max_load_reduction_pct=0.663313 moved_to_replicas=189289 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[32445, 32766, 32443, 32762, 33096, 32766, 32770, 33096] avg=32768.0000 max=33096 max_rank=4 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65203 max_load_reduction_pct=0.663313 moved_to_replicas=189289 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[2788, 32766, 6, 32762, 98299, 32766, 4, 62753] avg=32768.0000 max=98299 max_rank=4 min=4 min_rank=6 imbalance_ratio=2.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[2788, 32766, 6, 32762, 98299, 32766, 4, 62753] avg=32768.0000 max=98299 max_rank=4 min=4 min_rank=6 imbalance_ratio=2.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[2788, 32766, 6, 32762, 98299, 32766, 4, 62753] avg=32768.0000 max=98299 max_rank=4 min=4 min_rank=6 imbalance_ratio=2.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[32765, 33096, 32763, 32443, 32766, 32764, 32451, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=129573 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[32445, 32766, 32443, 32762, 33096, 32766, 32770, 33096] avg=32768.0000 max=33096 max_rank=4 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65203 max_load_reduction_pct=0.663313 moved_to_replicas=189289 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[32445, 32766, 32443, 32762, 33096, 32766, 32770, 33096] avg=32768.0000 max=33096 max_rank=4 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65203 max_load_reduction_pct=0.663313 moved_to_replicas=189289 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[32445, 32766, 32443, 32762, 33096, 32766, 32770, 33096] avg=32768.0000 max=33096 max_rank=4 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65203 max_load_reduction_pct=0.663313 moved_to_replicas=189289 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[32445, 32766, 32443, 32762, 33096, 32766, 32770, 33096] avg=32768.0000 max=33096 max_rank=4 min=32443 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65203 max_load_reduction_pct=0.663313 moved_to_replicas=189289 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[32765, 65536, 32763, 3, 32766, 32764, 28, 65519] avg=32768.0000 max=65536 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[32765, 65536, 32763, 3, 32766, 32764, 28, 65519] avg=32768.0000 max=65536 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[32765, 65536, 32763, 3, 32766, 32764, 28, 65519] avg=32768.0000 max=65536 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[32765, 33096, 32763, 32443, 32766, 32764, 32451, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=129573 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[32765, 33096, 32763, 32443, 32766, 32764, 32451, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=129573 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[32765, 65536, 32763, 3, 32766, 32764, 28, 65519] avg=32768.0000 max=65536 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[32765, 65536, 32763, 3, 32766, 32764, 28, 65519] avg=32768.0000 max=65536 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[32765, 65536, 32763, 3, 32766, 32764, 28, 65519] avg=32768.0000 max=65536 max_rank=1 min=3 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[32765, 33096, 32763, 32443, 32766, 32764, 32451, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=129573 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[32765, 33096, 32763, 32443, 32766, 32764, 32451, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=129573 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[32765, 33096, 32763, 32443, 32766, 32764, 32451, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=129573 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[32765, 33096, 32763, 32443, 32766, 32764, 32451, 33096] avg=32768.0000 max=33096 max_rank=1 min=32443 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=129573 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=32 total_expected_recv=1046885 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253312 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=32 total_expected_recv=1041393 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=259200 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=32 total_expected_recv=1046335 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254080 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=32 total_expected_recv=1052298 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=248576 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=32 total_expected_recv=1053700 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=247936 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=32 total_expected_recv=1049681 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=250496 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=32 total_expected_recv=1050261 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=250624 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=32 total_expected_recv=1048055 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253184 peak_demand=33096 required_capacity=1.010010 + [2026-08-30 05:10:28] iteration 5/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 534.6 | throughput per GPU (TFLOP/s/GPU): 623.6 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.039262E+01 | loss scale: 1.0 | grad norm: 16.276 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30472, 33096, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30472, 33096, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30472, 33096, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=40960 expected_recv=60944 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30472, 33096, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30472, 33096, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30472, 33096, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30472, 33096, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=2 budget=40960 expected_recv=60944 actual_valid_recv=61696 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=61696 slack_slots=20224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 30472, 33096, 33096, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=40960 expected_recv=63945 expected_overflow=0 exact_recv_per_rank=[33096, 32595, 31965, 32725, 32807, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[33096, 32595, 31965, 32725, 32807, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32595, 31965, 32725, 32807, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65319 expected_overflow=0 exact_recv_per_rank=[33096, 32595, 31965, 32725, 32807, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65398 expected_overflow=0 exact_recv_per_rank=[33096, 32595, 31965, 32725, 32807, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=40960 expected_recv=66157 expected_overflow=0 exact_recv_per_rank=[33096, 32595, 31965, 32725, 32807, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=2 budget=40960 expected_recv=63945 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32595, 31965, 32725, 32807, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65535 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65319 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65398 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=2 budget=40960 expected_recv=66157 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[33096, 32595, 31965, 32725, 32807, 32764, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=2 budget=40960 expected_recv=65550 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32782, 32778, 33096, 31782, 32416, 33098, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[32782, 32778, 33096, 31782, 32416, 33098, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=40960 expected_recv=65555 expected_overflow=0 exact_recv_per_rank=[32782, 32778, 33096, 31782, 32416, 33098, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=40960 expected_recv=65641 expected_overflow=0 exact_recv_per_rank=[32782, 32778, 33096, 31782, 32416, 33098, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=40960 expected_recv=63455 expected_overflow=0 exact_recv_per_rank=[32782, 32778, 33096, 31782, 32416, 33098, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32782, 32778, 33096, 31782, 32416, 33098, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32782, 32778, 33096, 31782, 32416, 33098, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=2 budget=40960 expected_recv=65555 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=2 budget=40960 expected_recv=65641 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=2 budget=40960 expected_recv=63455 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=40960 expected_recv=64867 expected_overflow=0 exact_recv_per_rank=[32782, 32778, 33096, 31782, 32416, 33098, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=2 budget=40960 expected_recv=64867 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=40960 expected_recv=64863 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32788, 32411, 32787, 32442, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=40960 expected_recv=65583 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32788, 32411, 32787, 32442, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=40960 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32788, 32411, 32787, 32442, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32788, 32411, 32787, 32442, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=2 budget=40960 expected_recv=64863 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=2 budget=40960 expected_recv=65583 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=2 budget=40960 expected_recv=65533 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=40960 expected_recv=64841 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32788, 32411, 32787, 32442, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=40960 expected_recv=65505 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32788, 32411, 32787, 32442, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32788, 32411, 32787, 32442, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=2 budget=40960 expected_recv=64841 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=2 budget=40960 expected_recv=65505 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=40960 expected_recv=65579 expected_overflow=0 exact_recv_per_rank=[32763, 33096, 32788, 32411, 32787, 32442, 32761, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=2 budget=40960 expected_recv=65579 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32785, 32460, 32385, 32795, 33096, 33096, 32431] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=40960 expected_recv=64768 expected_overflow=0 exact_recv_per_rank=[33096, 32785, 32460, 32385, 32795, 33096, 33096, 32431] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32785, 32460, 32385, 32795, 33096, 33096, 32431] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=40960 expected_recv=65524 expected_overflow=0 exact_recv_per_rank=[33096, 32785, 32460, 32385, 32795, 33096, 33096, 32431] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=2 budget=40960 expected_recv=64768 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=40960 expected_recv=64761 expected_overflow=0 exact_recv_per_rank=[33096, 32785, 32460, 32385, 32795, 33096, 33096, 32431] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=2 budget=40960 expected_recv=65524 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32785, 32460, 32385, 32795, 33096, 33096, 32431] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=40960 expected_recv=64872 expected_overflow=0 exact_recv_per_rank=[33096, 32785, 32460, 32385, 32795, 33096, 33096, 32431] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=2 budget=40960 expected_recv=64761 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=2 budget=40960 expected_recv=64872 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=40960 expected_recv=65787 expected_overflow=0 exact_recv_per_rank=[33096, 32785, 32460, 32385, 32795, 33096, 33096, 32431] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=2 budget=40960 expected_recv=65787 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 32770, 32780, 32080, 33096, 32452, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 32770, 32780, 32080, 33096, 32452, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 32770, 32780, 32080, 33096, 32452, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 32770, 32780, 32080, 33096, 32452, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65382 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 32770, 32780, 32080, 33096, 32452, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65536 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=40960 expected_recv=64911 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 32770, 32780, 32080, 33096, 32452, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 32770, 32780, 32080, 33096, 32452, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65537 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=2 budget=40960 expected_recv=65382 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=2 budget=40960 expected_recv=64911 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=40960 expected_recv=64346 expected_overflow=0 exact_recv_per_rank=[33096, 32774, 32770, 32780, 32080, 33096, 32452, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=2 budget=40960 expected_recv=64346 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=40960 expected_recv=64384 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 33096, 32507, 33096, 32226, 32591, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 33096, 32507, 33096, 32226, 32591, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 33096, 32507, 33096, 32226, 32591, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 33096, 32507, 33096, 32226, 32591, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=2 budget=40960 expected_recv=64384 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65237 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 33096, 32507, 33096, 32226, 32591, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65029 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 33096, 32507, 33096, 32226, 32591, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65539 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 33096, 32507, 33096, 32226, 32591, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65237 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65029 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=2 budget=40960 expected_recv=65523 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32778, 33096, 33096, 32507, 33096, 32226, 32591, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32372, 32815, 32768, 32767, 32768, 33096, 33096, 32462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[32372, 32815, 32768, 32767, 32768, 33096, 33096, 32462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=40960 expected_recv=64823 expected_overflow=0 exact_recv_per_rank=[32372, 32815, 32768, 32767, 32768, 33096, 33096, 32462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65664 expected_overflow=0 exact_recv_per_rank=[32372, 32815, 32768, 32767, 32768, 33096, 33096, 32462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65540 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=2 budget=40960 expected_recv=64823 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65664 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32372, 32815, 32768, 32767, 32768, 33096, 33096, 32462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65513 expected_overflow=0 exact_recv_per_rank=[32372, 32815, 32768, 32767, 32768, 33096, 33096, 32462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=40960 expected_recv=64830 expected_overflow=0 exact_recv_per_rank=[32372, 32815, 32768, 32767, 32768, 33096, 33096, 32462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65513 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=2 budget=40960 expected_recv=64830 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[32372, 32815, 32768, 32767, 32768, 33096, 33096, 32462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=2 budget=40960 expected_recv=65534 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65474 expected_overflow=0 exact_recv_per_rank=[33096, 32189, 32797, 32397, 32774, 32699, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65567 expected_overflow=0 exact_recv_per_rank=[33096, 32189, 32797, 32397, 32774, 32699, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32189, 32797, 32397, 32774, 32699, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65474 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65567 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=40960 expected_recv=64607 expected_overflow=0 exact_recv_per_rank=[33096, 32189, 32797, 32397, 32774, 32699, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=2 budget=40960 expected_recv=64607 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32189, 32797, 32397, 32774, 32699, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=40960 expected_recv=64519 expected_overflow=0 exact_recv_per_rank=[33096, 32189, 32797, 32397, 32774, 32699, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32189, 32797, 32397, 32774, 32699, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=2 budget=40960 expected_recv=64519 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[33096, 32189, 32797, 32397, 32774, 32699, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[32837, 33096, 32406, 32771, 32704, 32773, 32763, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=2 budget=40960 expected_recv=65545 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=40960 expected_recv=64777 expected_overflow=0 exact_recv_per_rank=[32837, 33096, 32406, 32771, 32704, 32773, 32763, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65550 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65671 expected_overflow=0 exact_recv_per_rank=[32837, 33096, 32406, 32771, 32704, 32773, 32763, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=2 budget=40960 expected_recv=64777 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32837, 33096, 32406, 32771, 32704, 32773, 32763, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65671 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[32837, 33096, 32406, 32771, 32704, 32773, 32763, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[32837, 33096, 32406, 32771, 32704, 32773, 32763, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65566 expected_overflow=0 exact_recv_per_rank=[32837, 33096, 32406, 32771, 32704, 32773, 32763, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65527 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65542 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65566 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65463 expected_overflow=0 exact_recv_per_rank=[32837, 33096, 32406, 32771, 32704, 32773, 32763, 32794] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32388, 31926, 33096, 32885, 32880, 33096, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=2 budget=40960 expected_recv=65463 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32388, 31926, 33096, 32885, 32880, 33096, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=40960 expected_recv=64779 expected_overflow=0 exact_recv_per_rank=[32388, 31926, 33096, 32885, 32880, 33096, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=40960 expected_recv=63631 expected_overflow=0 exact_recv_per_rank=[32388, 31926, 33096, 32885, 32880, 33096, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=2 budget=40960 expected_recv=64779 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=2 budget=40960 expected_recv=63631 actual_valid_recv=64128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65569 expected_overflow=0 exact_recv_per_rank=[32388, 31926, 33096, 32885, 32880, 33096, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=64128 slack_slots=17792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65757 expected_overflow=0 exact_recv_per_rank=[32388, 31926, 33096, 32885, 32880, 33096, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32388, 31926, 33096, 32885, 32880, 33096, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65569 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65757 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65976 expected_overflow=0 exact_recv_per_rank=[32388, 31926, 33096, 32885, 32880, 33096, 32777, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65554 expected_overflow=0 exact_recv_per_rank=[33096, 32716, 32771, 32455, 32464, 32775, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65558 expected_overflow=0 exact_recv_per_rank=[33096, 32716, 32771, 32455, 32464, 32775, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=2 budget=40960 expected_recv=65976 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32716, 32771, 32455, 32464, 32775, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65554 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65558 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65386 expected_overflow=0 exact_recv_per_rank=[33096, 32716, 32771, 32455, 32464, 32775, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65386 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[33096, 32716, 32771, 32455, 32464, 32775, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=40960 expected_recv=64968 expected_overflow=0 exact_recv_per_rank=[33096, 32716, 32771, 32455, 32464, 32775, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=2 budget=40960 expected_recv=65538 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32716, 32771, 32455, 32464, 32775, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=2 budget=40960 expected_recv=64968 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=40960 expected_recv=64900 expected_overflow=0 exact_recv_per_rank=[33096, 32716, 32771, 32455, 32464, 32775, 32771, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=40960 expected_recv=64880 expected_overflow=0 exact_recv_per_rank=[32785, 33098, 33095, 32754, 33096, 31783, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=40960 expected_recv=64867 expected_overflow=0 exact_recv_per_rank=[32785, 33098, 33095, 32754, 33096, 31783, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=2 budget=40960 expected_recv=64900 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65585 expected_overflow=0 exact_recv_per_rank=[32785, 33098, 33095, 32754, 33096, 31783, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=12 calls=2 budget=40960 expected_recv=64880 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32785, 33098, 33095, 32754, 33096, 31783, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=2 budget=40960 expected_recv=64867 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65585 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=12 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65528 expected_overflow=0 exact_recv_per_rank=[32785, 33098, 33095, 32754, 33096, 31783, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65500 expected_overflow=0 exact_recv_per_rank=[32785, 33098, 33095, 32754, 33096, 31783, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65528 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[32785, 33098, 33095, 32754, 33096, 31783, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65500 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=12 calls=2 budget=40960 expected_recv=65543 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32785, 33098, 33095, 32754, 33096, 31783, 32766, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65687 expected_overflow=0 exact_recv_per_rank=[32395, 32780, 33096, 32766, 32442, 32793, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32395, 32780, 33096, 32766, 32442, 32793, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=40960 expected_recv=64712 expected_overflow=0 exact_recv_per_rank=[32395, 32780, 33096, 32766, 32442, 32793, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65687 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65563 expected_overflow=0 exact_recv_per_rank=[32395, 32780, 33096, 32766, 32442, 32793, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=13 calls=2 budget=40960 expected_recv=64712 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[32395, 32780, 33096, 32766, 32442, 32793, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65563 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[32395, 32780, 33096, 32766, 32442, 32793, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65526 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32395, 32780, 33096, 32766, 32442, 32793, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=13 calls=2 budget=40960 expected_recv=65531 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=40960 expected_recv=64885 expected_overflow=0 exact_recv_per_rank=[32395, 32780, 33096, 32766, 32442, 32793, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[32765, 32435, 32789, 32758, 33096, 32782, 33096, 32423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65584 expected_overflow=0 exact_recv_per_rank=[32765, 32435, 32789, 32758, 33096, 32782, 33096, 32423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=2 budget=40960 expected_recv=64885 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65528 expected_overflow=0 exact_recv_per_rank=[32765, 32435, 32789, 32758, 33096, 32782, 33096, 32423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65543 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65584 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=40960 expected_recv=64883 expected_overflow=0 exact_recv_per_rank=[32765, 32435, 32789, 32758, 33096, 32782, 33096, 32423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65528 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32765, 32435, 32789, 32758, 33096, 32782, 33096, 32423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=2 budget=40960 expected_recv=64883 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65521 expected_overflow=0 exact_recv_per_rank=[32765, 32435, 32789, 32758, 33096, 32782, 33096, 32423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=14 calls=2 budget=40960 expected_recv=65521 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=40960 expected_recv=64845 expected_overflow=0 exact_recv_per_rank=[32765, 32435, 32789, 32758, 33096, 32782, 33096, 32423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=14 calls=2 budget=40960 expected_recv=64845 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32765, 32435, 32789, 32758, 33096, 32782, 33096, 32423] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=40960 expected_recv=64869 expected_overflow=0 exact_recv_per_rank=[33096, 32801, 32417, 32435, 32768, 32435, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=40960 expected_recv=64826 expected_overflow=0 exact_recv_per_rank=[33096, 32801, 32417, 32435, 32768, 32435, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=15 calls=2 budget=40960 expected_recv=64869 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32801, 32417, 32435, 32768, 32435, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=15 calls=2 budget=40960 expected_recv=64826 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=40960 expected_recv=65597 expected_overflow=0 exact_recv_per_rank=[33096, 32801, 32417, 32435, 32768, 32435, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=40960 expected_recv=64884 expected_overflow=0 exact_recv_per_rank=[33096, 32801, 32417, 32435, 32768, 32435, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32801, 32417, 32435, 32768, 32435, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=15 calls=2 budget=40960 expected_recv=65597 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=15 calls=2 budget=40960 expected_recv=64884 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32801, 32417, 32435, 32768, 32435, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=40960 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[33096, 32801, 32417, 32435, 32768, 32435, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=2 budget=40960 expected_recv=65536 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[25431, 18645, 29532, 46842, 49650, 29092, 21137, 41815] avg=32768.0000 max=49650 max_rank=4 min=18645 min_rank=1 imbalance_ratio=1.515198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[25431, 18645, 29532, 46842, 49650, 29092, 21137, 41815] avg=32768.0000 max=49650 max_rank=4 min=18645 min_rank=1 imbalance_ratio=1.515198 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[25431, 18645, 29532, 46842, 49650, 29092, 21137, 41815] avg=32768.0000 max=49650 max_rank=4 min=18645 min_rank=1 imbalance_ratio=1.515198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[25431, 18645, 29532, 46842, 49650, 29092, 21137, 41815] avg=32768.0000 max=49650 max_rank=4 min=18645 min_rank=1 imbalance_ratio=1.515198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[25431, 18645, 29532, 46842, 49650, 29092, 21137, 41815] avg=32768.0000 max=49650 max_rank=4 min=18645 min_rank=1 imbalance_ratio=1.515198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[25431, 18645, 29532, 46842, 49650, 29092, 21137, 41815] avg=32768.0000 max=49650 max_rank=4 min=18645 min_rank=1 imbalance_ratio=1.515198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 30472, 33096, 33096, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16553 max_load_reduction_pct=0.333394 moved_to_replicas=75781 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 30472, 33096, 33096, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16553 max_load_reduction_pct=0.333394 moved_to_replicas=75781 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[25431, 18645, 29532, 46842, 49650, 29092, 21137, 41815] avg=32768.0000 max=49650 max_rank=4 min=18645 min_rank=1 imbalance_ratio=1.515198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 30472, 33096, 33096, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16553 max_load_reduction_pct=0.333394 moved_to_replicas=75781 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 30472, 33096, 33096, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16553 max_load_reduction_pct=0.333394 moved_to_replicas=75781 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 30472, 33096, 33096, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16553 max_load_reduction_pct=0.333394 moved_to_replicas=75781 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[25431, 18645, 29532, 46842, 49650, 29092, 21137, 41815] avg=32768.0000 max=49650 max_rank=4 min=18645 min_rank=1 imbalance_ratio=1.515198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 30472, 33096, 33096, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16553 max_load_reduction_pct=0.333394 moved_to_replicas=75781 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 30472, 33096, 33096, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16553 max_load_reduction_pct=0.333394 moved_to_replicas=75781 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[62828, 32595, 207, 32725, 32807, 32764, 33585, 34633] avg=32768.0000 max=62828 max_rank=0 min=207 min_rank=2 imbalance_ratio=1.917358 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[62828, 32595, 207, 32725, 32807, 32764, 33585, 34633] avg=32768.0000 max=62828 max_rank=0 min=207 min_rank=2 imbalance_ratio=1.917358 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[62828, 32595, 207, 32725, 32807, 32764, 33585, 34633] avg=32768.0000 max=62828 max_rank=0 min=207 min_rank=2 imbalance_ratio=1.917358 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 30472, 33096, 33096, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=16553 max_load_reduction_pct=0.333394 moved_to_replicas=75781 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[62828, 32595, 207, 32725, 32807, 32764, 33585, 34633] avg=32768.0000 max=62828 max_rank=0 min=207 min_rank=2 imbalance_ratio=1.917358 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[62828, 32595, 207, 32725, 32807, 32764, 33585, 34633] avg=32768.0000 max=62828 max_rank=0 min=207 min_rank=2 imbalance_ratio=1.917358 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[62828, 32595, 207, 32725, 32807, 32764, 33585, 34633] avg=32768.0000 max=62828 max_rank=0 min=207 min_rank=2 imbalance_ratio=1.917358 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32595, 31965, 32725, 32807, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31965 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=29732 max_load_reduction_pct=0.473228 moved_to_replicas=63335 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32595, 31965, 32725, 32807, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31965 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=29732 max_load_reduction_pct=0.473228 moved_to_replicas=63335 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32595, 31965, 32725, 32807, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31965 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=29732 max_load_reduction_pct=0.473228 moved_to_replicas=63335 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[62828, 32595, 207, 32725, 32807, 32764, 33585, 34633] avg=32768.0000 max=62828 max_rank=0 min=207 min_rank=2 imbalance_ratio=1.917358 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32595, 31965, 32725, 32807, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31965 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=29732 max_load_reduction_pct=0.473228 moved_to_replicas=63335 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32595, 31965, 32725, 32807, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31965 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=29732 max_load_reduction_pct=0.473228 moved_to_replicas=63335 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32595, 31965, 32725, 32807, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31965 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=29732 max_load_reduction_pct=0.473228 moved_to_replicas=63335 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[62828, 32595, 207, 32725, 32807, 32764, 33585, 34633] avg=32768.0000 max=62828 max_rank=0 min=207 min_rank=2 imbalance_ratio=1.917358 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32595, 31965, 32725, 32807, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31965 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=29732 max_load_reduction_pct=0.473228 moved_to_replicas=63335 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[32782, 32778, 53791, 25525, 2, 37584, 14172, 65510] avg=32768.0000 max=65510 max_rank=7 min=2 min_rank=4 imbalance_ratio=1.999207 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[32782, 32778, 53791, 25525, 2, 37584, 14172, 65510] avg=32768.0000 max=65510 max_rank=7 min=2 min_rank=4 imbalance_ratio=1.999207 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[32782, 32778, 53791, 25525, 2, 37584, 14172, 65510] avg=32768.0000 max=65510 max_rank=7 min=2 min_rank=4 imbalance_ratio=1.999207 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[32782, 32778, 53791, 25525, 2, 37584, 14172, 65510] avg=32768.0000 max=65510 max_rank=7 min=2 min_rank=4 imbalance_ratio=1.999207 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[32782, 32778, 53791, 25525, 2, 37584, 14172, 65510] avg=32768.0000 max=65510 max_rank=7 min=2 min_rank=4 imbalance_ratio=1.999207 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[33096, 32595, 31965, 32725, 32807, 32764, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31965 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=29732 max_load_reduction_pct=0.473228 moved_to_replicas=63335 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[32782, 32778, 53791, 25525, 2, 37584, 14172, 65510] avg=32768.0000 max=65510 max_rank=7 min=2 min_rank=4 imbalance_ratio=1.999207 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[32782, 32778, 33096, 31782, 32416, 33098, 33096, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32412 max_load_reduction_pct=0.494764 moved_to_replicas=119594 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[32782, 32778, 33096, 31782, 32416, 33098, 33096, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32412 max_load_reduction_pct=0.494764 moved_to_replicas=119594 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[32782, 32778, 33096, 31782, 32416, 33098, 33096, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32412 max_load_reduction_pct=0.494764 moved_to_replicas=119594 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[32782, 32778, 33096, 31782, 32416, 33098, 33096, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32412 max_load_reduction_pct=0.494764 moved_to_replicas=119594 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[32782, 32778, 53791, 25525, 2, 37584, 14172, 65510] avg=32768.0000 max=65510 max_rank=7 min=2 min_rank=4 imbalance_ratio=1.999207 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[32782, 32778, 33096, 31782, 32416, 33098, 33096, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32412 max_load_reduction_pct=0.494764 moved_to_replicas=119594 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[32782, 32778, 33096, 31782, 32416, 33098, 33096, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32412 max_load_reduction_pct=0.494764 moved_to_replicas=119594 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[32782, 32778, 53791, 25525, 2, 37584, 14172, 65510] avg=32768.0000 max=65510 max_rank=7 min=2 min_rank=4 imbalance_ratio=1.999207 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[32763, 65492, 32788, 15, 32787, 73, 32761, 65465] avg=32768.0000 max=65492 max_rank=1 min=15 min_rank=3 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[32782, 32778, 33096, 31782, 32416, 33098, 33096, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32412 max_load_reduction_pct=0.494764 moved_to_replicas=119594 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[32763, 65492, 32788, 15, 32787, 73, 32761, 65465] avg=32768.0000 max=65492 max_rank=1 min=15 min_rank=3 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[32763, 65492, 32788, 15, 32787, 73, 32761, 65465] avg=32768.0000 max=65492 max_rank=1 min=15 min_rank=3 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[32763, 65492, 32788, 15, 32787, 73, 32761, 65465] avg=32768.0000 max=65492 max_rank=1 min=15 min_rank=3 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[32782, 32778, 33096, 31782, 32416, 33098, 33096, 33096] avg=32768.0000 max=33098 max_rank=5 min=31782 min_rank=3 imbalance_ratio=1.010071 max_load_reduction=32412 max_load_reduction_pct=0.494764 moved_to_replicas=119594 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[32763, 65492, 32788, 15, 32787, 73, 32761, 65465] avg=32768.0000 max=65492 max_rank=1 min=15 min_rank=3 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[32763, 65492, 32788, 15, 32787, 73, 32761, 65465] avg=32768.0000 max=65492 max_rank=1 min=15 min_rank=3 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[32763, 33096, 32788, 32411, 32787, 32442, 32761, 33096] avg=32768.0000 max=33096 max_rank=1 min=32411 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=129558 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[32763, 33096, 32788, 32411, 32787, 32442, 32761, 33096] avg=32768.0000 max=33096 max_rank=1 min=32411 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=129558 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[32763, 33096, 32788, 32411, 32787, 32442, 32761, 33096] avg=32768.0000 max=33096 max_rank=1 min=32411 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=129558 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[32763, 33096, 32788, 32411, 32787, 32442, 32761, 33096] avg=32768.0000 max=33096 max_rank=1 min=32411 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=129558 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[32763, 65492, 32788, 15, 32787, 73, 32761, 65465] avg=32768.0000 max=65492 max_rank=1 min=15 min_rank=3 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[32763, 33096, 32788, 32411, 32787, 32442, 32761, 33096] avg=32768.0000 max=33096 max_rank=1 min=32411 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=129558 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[32763, 33096, 32788, 32411, 32787, 32442, 32761, 33096] avg=32768.0000 max=33096 max_rank=1 min=32411 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=129558 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[32763, 65492, 32788, 15, 32787, 73, 32761, 65465] avg=32768.0000 max=65492 max_rank=1 min=15 min_rank=3 imbalance_ratio=1.998657 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[65217, 32785, 339, 12, 32795, 65469, 65521, 6] avg=32768.0000 max=65521 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[32763, 33096, 32788, 32411, 32787, 32442, 32761, 33096] avg=32768.0000 max=33096 max_rank=1 min=32411 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=129558 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[65217, 32785, 339, 12, 32795, 65469, 65521, 6] avg=32768.0000 max=65521 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[65217, 32785, 339, 12, 32795, 65469, 65521, 6] avg=32768.0000 max=65521 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[65217, 32785, 339, 12, 32795, 65469, 65521, 6] avg=32768.0000 max=65521 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[32763, 33096, 32788, 32411, 32787, 32442, 32761, 33096] avg=32768.0000 max=33096 max_rank=1 min=32411 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32396 max_load_reduction_pct=0.494656 moved_to_replicas=129558 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[65217, 32785, 339, 12, 32795, 65469, 65521, 6] avg=32768.0000 max=65521 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[65217, 32785, 339, 12, 32795, 65469, 65521, 6] avg=32768.0000 max=65521 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32785, 32460, 32385, 32795, 33096, 33096, 32431] avg=32768.0000 max=33096 max_rank=0 min=32385 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=193566 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32785, 32460, 32385, 32795, 33096, 33096, 32431] avg=32768.0000 max=33096 max_rank=0 min=32385 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=193566 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32785, 32460, 32385, 32795, 33096, 33096, 32431] avg=32768.0000 max=33096 max_rank=0 min=32385 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=193566 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32785, 32460, 32385, 32795, 33096, 33096, 32431] avg=32768.0000 max=33096 max_rank=0 min=32385 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=193566 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32785, 32460, 32385, 32795, 33096, 33096, 32431] avg=32768.0000 max=33096 max_rank=0 min=32385 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=193566 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32785, 32460, 32385, 32795, 33096, 33096, 32431] avg=32768.0000 max=33096 max_rank=0 min=32385 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=193566 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[65217, 32785, 339, 12, 32795, 65469, 65521, 6] avg=32768.0000 max=65521 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[65217, 32785, 339, 12, 32795, 65469, 65521, 6] avg=32768.0000 max=65521 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.999542 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[66486, 12, 32770, 32780, 22, 63806, 32452, 33816] avg=32768.0000 max=66486 max_rank=0 min=12 min_rank=1 imbalance_ratio=2.028992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[66486, 12, 32770, 32780, 22, 63806, 32452, 33816] avg=32768.0000 max=66486 max_rank=0 min=12 min_rank=1 imbalance_ratio=2.028992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[66486, 12, 32770, 32780, 22, 63806, 32452, 33816] avg=32768.0000 max=66486 max_rank=0 min=12 min_rank=1 imbalance_ratio=2.028992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[66486, 12, 32770, 32780, 22, 63806, 32452, 33816] avg=32768.0000 max=66486 max_rank=0 min=12 min_rank=1 imbalance_ratio=2.028992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32785, 32460, 32385, 32795, 33096, 33096, 32431] avg=32768.0000 max=33096 max_rank=0 min=32385 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=193566 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32785, 32460, 32385, 32795, 33096, 33096, 32431] avg=32768.0000 max=33096 max_rank=0 min=32385 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32425 max_load_reduction_pct=0.494880 moved_to_replicas=193566 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[66486, 12, 32770, 32780, 22, 63806, 32452, 33816] avg=32768.0000 max=66486 max_rank=0 min=12 min_rank=1 imbalance_ratio=2.028992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[66486, 12, 32770, 32780, 22, 63806, 32452, 33816] avg=32768.0000 max=66486 max_rank=0 min=12 min_rank=1 imbalance_ratio=2.028992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[33096, 32774, 32770, 32780, 32080, 33096, 32452, 33096] avg=32768.0000 max=33096 max_rank=0 min=32080 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33390 max_load_reduction_pct=0.502211 moved_to_replicas=129823 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[33096, 32774, 32770, 32780, 32080, 33096, 32452, 33096] avg=32768.0000 max=33096 max_rank=0 min=32080 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33390 max_load_reduction_pct=0.502211 moved_to_replicas=129823 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[33096, 32774, 32770, 32780, 32080, 33096, 32452, 33096] avg=32768.0000 max=33096 max_rank=0 min=32080 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33390 max_load_reduction_pct=0.502211 moved_to_replicas=129823 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[33096, 32774, 32770, 32780, 32080, 33096, 32452, 33096] avg=32768.0000 max=33096 max_rank=0 min=32080 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33390 max_load_reduction_pct=0.502211 moved_to_replicas=129823 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[33096, 32774, 32770, 32780, 32080, 33096, 32452, 33096] avg=32768.0000 max=33096 max_rank=0 min=32080 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33390 max_load_reduction_pct=0.502211 moved_to_replicas=129823 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[33096, 32774, 32770, 32780, 32080, 33096, 32452, 33096] avg=32768.0000 max=33096 max_rank=0 min=32080 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33390 max_load_reduction_pct=0.502211 moved_to_replicas=129823 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[66486, 12, 32770, 32780, 22, 63806, 32452, 33816] avg=32768.0000 max=66486 max_rank=0 min=12 min_rank=1 imbalance_ratio=2.028992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[66486, 12, 32770, 32780, 22, 63806, 32452, 33816] avg=32768.0000 max=66486 max_rank=0 min=12 min_rank=1 imbalance_ratio=2.028992 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[32778, 65498, 65520, 83, 66016, 32226, 19, 4] avg=32768.0000 max=66016 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.014648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[32778, 65498, 65520, 83, 66016, 32226, 19, 4] avg=32768.0000 max=66016 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.014648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[32778, 65498, 65520, 83, 66016, 32226, 19, 4] avg=32768.0000 max=66016 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.014648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[32778, 65498, 65520, 83, 66016, 32226, 19, 4] avg=32768.0000 max=66016 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.014648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[33096, 32774, 32770, 32780, 32080, 33096, 32452, 33096] avg=32768.0000 max=33096 max_rank=0 min=32080 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33390 max_load_reduction_pct=0.502211 moved_to_replicas=129823 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[33096, 32774, 32770, 32780, 32080, 33096, 32452, 33096] avg=32768.0000 max=33096 max_rank=0 min=32080 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=33390 max_load_reduction_pct=0.502211 moved_to_replicas=129823 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[32778, 65498, 65520, 83, 66016, 32226, 19, 4] avg=32768.0000 max=66016 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.014648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[32778, 65498, 65520, 83, 66016, 32226, 19, 4] avg=32768.0000 max=66016 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.014648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[32778, 33096, 33096, 32507, 33096, 32226, 32591, 32754] avg=32768.0000 max=33096 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32920 max_load_reduction_pct=0.498667 moved_to_replicas=195455 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[32778, 33096, 33096, 32507, 33096, 32226, 32591, 32754] avg=32768.0000 max=33096 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32920 max_load_reduction_pct=0.498667 moved_to_replicas=195455 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[32778, 33096, 33096, 32507, 33096, 32226, 32591, 32754] avg=32768.0000 max=33096 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32920 max_load_reduction_pct=0.498667 moved_to_replicas=195455 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[32778, 33096, 33096, 32507, 33096, 32226, 32591, 32754] avg=32768.0000 max=33096 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32920 max_load_reduction_pct=0.498667 moved_to_replicas=195455 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[32778, 33096, 33096, 32507, 33096, 32226, 32591, 32754] avg=32768.0000 max=33096 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32920 max_load_reduction_pct=0.498667 moved_to_replicas=195455 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[32778, 33096, 33096, 32507, 33096, 32226, 32591, 32754] avg=32768.0000 max=33096 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32920 max_load_reduction_pct=0.498667 moved_to_replicas=195455 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[32778, 65498, 65520, 83, 66016, 32226, 19, 4] avg=32768.0000 max=66016 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.014648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[32778, 65498, 65520, 83, 66016, 32226, 19, 4] avg=32768.0000 max=66016 max_rank=4 min=4 min_rank=7 imbalance_ratio=2.014648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[21, 32815, 32768, 32767, 32768, 65543, 65447, 15] avg=32768.0000 max=65543 max_rank=5 min=15 min_rank=7 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[21, 32815, 32768, 32767, 32768, 65543, 65447, 15] avg=32768.0000 max=65543 max_rank=5 min=15 min_rank=7 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[21, 32815, 32768, 32767, 32768, 65543, 65447, 15] avg=32768.0000 max=65543 max_rank=5 min=15 min_rank=7 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[21, 32815, 32768, 32767, 32768, 65543, 65447, 15] avg=32768.0000 max=65543 max_rank=5 min=15 min_rank=7 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[32778, 33096, 33096, 32507, 33096, 32226, 32591, 32754] avg=32768.0000 max=33096 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32920 max_load_reduction_pct=0.498667 moved_to_replicas=195455 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[32778, 33096, 33096, 32507, 33096, 32226, 32591, 32754] avg=32768.0000 max=33096 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32920 max_load_reduction_pct=0.498667 moved_to_replicas=195455 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[21, 32815, 32768, 32767, 32768, 65543, 65447, 15] avg=32768.0000 max=65543 max_rank=5 min=15 min_rank=7 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[21, 32815, 32768, 32767, 32768, 65543, 65447, 15] avg=32768.0000 max=65543 max_rank=5 min=15 min_rank=7 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[32372, 32815, 32768, 32767, 32768, 33096, 33096, 32462] avg=32768.0000 max=33096 max_rank=5 min=32372 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=129560 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[32372, 32815, 32768, 32767, 32768, 33096, 33096, 32462] avg=32768.0000 max=33096 max_rank=5 min=32372 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=129560 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[32372, 32815, 32768, 32767, 32768, 33096, 33096, 32462] avg=32768.0000 max=33096 max_rank=5 min=32372 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=129560 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[32372, 32815, 32768, 32767, 32768, 33096, 33096, 32462] avg=32768.0000 max=33096 max_rank=5 min=32372 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=129560 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[32372, 32815, 32768, 32767, 32768, 33096, 33096, 32462] avg=32768.0000 max=33096 max_rank=5 min=32372 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=129560 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[32372, 32815, 32768, 32767, 32768, 33096, 33096, 32462] avg=32768.0000 max=33096 max_rank=5 min=32372 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=129560 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[21, 32815, 32768, 32767, 32768, 65543, 65447, 15] avg=32768.0000 max=65543 max_rank=5 min=15 min_rank=7 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[21, 32815, 32768, 32767, 32768, 65543, 65447, 15] avg=32768.0000 max=65543 max_rank=5 min=15 min_rank=7 imbalance_ratio=2.000214 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[65478, 19, 32797, 15, 32774, 13, 65266, 65782] avg=32768.0000 max=65782 max_rank=7 min=13 min_rank=5 imbalance_ratio=2.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[65478, 19, 32797, 15, 32774, 13, 65266, 65782] avg=32768.0000 max=65782 max_rank=7 min=13 min_rank=5 imbalance_ratio=2.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[65478, 19, 32797, 15, 32774, 13, 65266, 65782] avg=32768.0000 max=65782 max_rank=7 min=13 min_rank=5 imbalance_ratio=2.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[32372, 32815, 32768, 32767, 32768, 33096, 33096, 32462] avg=32768.0000 max=33096 max_rank=5 min=32372 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=129560 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[32372, 32815, 32768, 32767, 32768, 33096, 33096, 32462] avg=32768.0000 max=33096 max_rank=5 min=32372 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32447 max_load_reduction_pct=0.495049 moved_to_replicas=129560 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[65478, 19, 32797, 15, 32774, 13, 65266, 65782] avg=32768.0000 max=65782 max_rank=7 min=13 min_rank=5 imbalance_ratio=2.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[65478, 19, 32797, 15, 32774, 13, 65266, 65782] avg=32768.0000 max=65782 max_rank=7 min=13 min_rank=5 imbalance_ratio=2.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[33096, 32189, 32797, 32397, 32774, 32699, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32686 max_load_reduction_pct=0.496884 moved_to_replicas=194487 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[65478, 19, 32797, 15, 32774, 13, 65266, 65782] avg=32768.0000 max=65782 max_rank=7 min=13 min_rank=5 imbalance_ratio=2.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[33096, 32189, 32797, 32397, 32774, 32699, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32686 max_load_reduction_pct=0.496884 moved_to_replicas=194487 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[33096, 32189, 32797, 32397, 32774, 32699, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32686 max_load_reduction_pct=0.496884 moved_to_replicas=194487 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[33096, 32189, 32797, 32397, 32774, 32699, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32686 max_load_reduction_pct=0.496884 moved_to_replicas=194487 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[33096, 32189, 32797, 32397, 32774, 32699, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32686 max_load_reduction_pct=0.496884 moved_to_replicas=194487 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[33096, 32189, 32797, 32397, 32774, 32699, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32686 max_load_reduction_pct=0.496884 moved_to_replicas=194487 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[65478, 19, 32797, 15, 32774, 13, 65266, 65782] avg=32768.0000 max=65782 max_rank=7 min=13 min_rank=5 imbalance_ratio=2.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[65478, 19, 32797, 15, 32774, 13, 65266, 65782] avg=32768.0000 max=65782 max_rank=7 min=13 min_rank=5 imbalance_ratio=2.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[32837, 98234, 27, 32771, 32704, 14, 32763, 32794] avg=32768.0000 max=98234 max_rank=1 min=14 min_rank=5 imbalance_ratio=2.997864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[32837, 98234, 27, 32771, 32704, 14, 32763, 32794] avg=32768.0000 max=98234 max_rank=1 min=14 min_rank=5 imbalance_ratio=2.997864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[32837, 98234, 27, 32771, 32704, 14, 32763, 32794] avg=32768.0000 max=98234 max_rank=1 min=14 min_rank=5 imbalance_ratio=2.997864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[33096, 32189, 32797, 32397, 32774, 32699, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32686 max_load_reduction_pct=0.496884 moved_to_replicas=194487 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[33096, 32189, 32797, 32397, 32774, 32699, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32189 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32686 max_load_reduction_pct=0.496884 moved_to_replicas=194487 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[32837, 98234, 27, 32771, 32704, 14, 32763, 32794] avg=32768.0000 max=98234 max_rank=1 min=14 min_rank=5 imbalance_ratio=2.997864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[32837, 98234, 27, 32771, 32704, 14, 32763, 32794] avg=32768.0000 max=98234 max_rank=1 min=14 min_rank=5 imbalance_ratio=2.997864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[32837, 98234, 27, 32771, 32704, 14, 32763, 32794] avg=32768.0000 max=98234 max_rank=1 min=14 min_rank=5 imbalance_ratio=2.997864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[32837, 33096, 32406, 32771, 32704, 32773, 32763, 32794] avg=32768.0000 max=33096 max_rank=1 min=32406 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65138 max_load_reduction_pct=0.663090 moved_to_replicas=130230 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[32837, 33096, 32406, 32771, 32704, 32773, 32763, 32794] avg=32768.0000 max=33096 max_rank=1 min=32406 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65138 max_load_reduction_pct=0.663090 moved_to_replicas=130230 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[32837, 33096, 32406, 32771, 32704, 32773, 32763, 32794] avg=32768.0000 max=33096 max_rank=1 min=32406 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65138 max_load_reduction_pct=0.663090 moved_to_replicas=130230 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[32837, 33096, 32406, 32771, 32704, 32773, 32763, 32794] avg=32768.0000 max=33096 max_rank=1 min=32406 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65138 max_load_reduction_pct=0.663090 moved_to_replicas=130230 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[32837, 33096, 32406, 32771, 32704, 32773, 32763, 32794] avg=32768.0000 max=33096 max_rank=1 min=32406 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65138 max_load_reduction_pct=0.663090 moved_to_replicas=130230 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[32837, 33096, 32406, 32771, 32704, 32773, 32763, 32794] avg=32768.0000 max=33096 max_rank=1 min=32406 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65138 max_load_reduction_pct=0.663090 moved_to_replicas=130230 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[32837, 98234, 27, 32771, 32704, 14, 32763, 32794] avg=32768.0000 max=98234 max_rank=1 min=14 min_rank=5 imbalance_ratio=2.997864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[32837, 98234, 27, 32771, 32704, 14, 32763, 32794] avg=32768.0000 max=98234 max_rank=1 min=14 min_rank=5 imbalance_ratio=2.997864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[11, 31656, 65040, 32885, 936, 65473, 32777, 33366] avg=32768.0000 max=65473 max_rank=5 min=11 min_rank=0 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[11, 31656, 65040, 32885, 936, 65473, 32777, 33366] avg=32768.0000 max=65473 max_rank=5 min=11 min_rank=0 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[11, 31656, 65040, 32885, 936, 65473, 32777, 33366] avg=32768.0000 max=65473 max_rank=5 min=11 min_rank=0 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[32837, 33096, 32406, 32771, 32704, 32773, 32763, 32794] avg=32768.0000 max=33096 max_rank=1 min=32406 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65138 max_load_reduction_pct=0.663090 moved_to_replicas=130230 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[32837, 33096, 32406, 32771, 32704, 32773, 32763, 32794] avg=32768.0000 max=33096 max_rank=1 min=32406 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65138 max_load_reduction_pct=0.663090 moved_to_replicas=130230 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[11, 31656, 65040, 32885, 936, 65473, 32777, 33366] avg=32768.0000 max=65473 max_rank=5 min=11 min_rank=0 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[11, 31656, 65040, 32885, 936, 65473, 32777, 33366] avg=32768.0000 max=65473 max_rank=5 min=11 min_rank=0 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[11, 31656, 65040, 32885, 936, 65473, 32777, 33366] avg=32768.0000 max=65473 max_rank=5 min=11 min_rank=0 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[32388, 31926, 33096, 32885, 32880, 33096, 32777, 33096] avg=32768.0000 max=33096 max_rank=2 min=31926 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=129256 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[32388, 31926, 33096, 32885, 32880, 33096, 32777, 33096] avg=32768.0000 max=33096 max_rank=2 min=31926 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=129256 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[32388, 31926, 33096, 32885, 32880, 33096, 32777, 33096] avg=32768.0000 max=33096 max_rank=2 min=31926 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=129256 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[32388, 31926, 33096, 32885, 32880, 33096, 32777, 33096] avg=32768.0000 max=33096 max_rank=2 min=31926 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=129256 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[32388, 31926, 33096, 32885, 32880, 33096, 32777, 33096] avg=32768.0000 max=33096 max_rank=2 min=31926 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=129256 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[32388, 31926, 33096, 32885, 32880, 33096, 32777, 33096] avg=32768.0000 max=33096 max_rank=2 min=31926 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=129256 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[11, 31656, 65040, 32885, 936, 65473, 32777, 33366] avg=32768.0000 max=65473 max_rank=5 min=11 min_rank=0 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[11, 31656, 65040, 32885, 936, 65473, 32777, 33366] avg=32768.0000 max=65473 max_rank=5 min=11 min_rank=0 imbalance_ratio=1.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[65319, 32716, 32771, 232, 46, 32775, 32771, 65514] avg=32768.0000 max=65514 max_rank=7 min=46 min_rank=4 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[65319, 32716, 32771, 232, 46, 32775, 32771, 65514] avg=32768.0000 max=65514 max_rank=7 min=46 min_rank=4 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[65319, 32716, 32771, 232, 46, 32775, 32771, 65514] avg=32768.0000 max=65514 max_rank=7 min=46 min_rank=4 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[32388, 31926, 33096, 32885, 32880, 33096, 32777, 33096] avg=32768.0000 max=33096 max_rank=2 min=31926 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=129256 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[65319, 32716, 32771, 232, 46, 32775, 32771, 65514] avg=32768.0000 max=65514 max_rank=7 min=46 min_rank=4 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[32388, 31926, 33096, 32885, 32880, 33096, 32777, 33096] avg=32768.0000 max=33096 max_rank=2 min=31926 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32377 max_load_reduction_pct=0.494509 moved_to_replicas=129256 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[65319, 32716, 32771, 232, 46, 32775, 32771, 65514] avg=32768.0000 max=65514 max_rank=7 min=46 min_rank=4 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[65319, 32716, 32771, 232, 46, 32775, 32771, 65514] avg=32768.0000 max=65514 max_rank=7 min=46 min_rank=4 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[33096, 32716, 32771, 32455, 32464, 32775, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32455 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=129213 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[33096, 32716, 32771, 32455, 32464, 32775, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32455 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=129213 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[33096, 32716, 32771, 32455, 32464, 32775, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32455 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=129213 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[33096, 32716, 32771, 32455, 32464, 32775, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32455 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=129213 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[33096, 32716, 32771, 32455, 32464, 32775, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32455 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=129213 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[65319, 32716, 32771, 232, 46, 32775, 32771, 65514] avg=32768.0000 max=65514 max_rank=7 min=46 min_rank=4 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[33096, 32716, 32771, 32455, 32464, 32775, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32455 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=129213 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[65319, 32716, 32771, 232, 46, 32775, 32771, 65514] avg=32768.0000 max=65514 max_rank=7 min=46 min_rank=4 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[39, 80008, 121, 32754, 65504, 18185, 32766, 32767] avg=32768.0000 max=80008 max_rank=1 min=39 min_rank=0 imbalance_ratio=2.441650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[39, 80008, 121, 32754, 65504, 18185, 32766, 32767] avg=32768.0000 max=80008 max_rank=1 min=39 min_rank=0 imbalance_ratio=2.441650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[39, 80008, 121, 32754, 65504, 18185, 32766, 32767] avg=32768.0000 max=80008 max_rank=1 min=39 min_rank=0 imbalance_ratio=2.441650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[33096, 32716, 32771, 32455, 32464, 32775, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32455 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=129213 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[39, 80008, 121, 32754, 65504, 18185, 32766, 32767] avg=32768.0000 max=80008 max_rank=1 min=39 min_rank=0 imbalance_ratio=2.441650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[33096, 32716, 32771, 32455, 32464, 32775, 32771, 33096] avg=32768.0000 max=33096 max_rank=0 min=32455 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=129213 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[39, 80008, 121, 32754, 65504, 18185, 32766, 32767] avg=32768.0000 max=80008 max_rank=1 min=39 min_rank=0 imbalance_ratio=2.441650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[39, 80008, 121, 32754, 65504, 18185, 32766, 32767] avg=32768.0000 max=80008 max_rank=1 min=39 min_rank=0 imbalance_ratio=2.441650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[32785, 33098, 33095, 32754, 33096, 31783, 32766, 32767] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=46910 max_load_reduction_pct=0.586316 moved_to_replicas=163162 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[32785, 33098, 33095, 32754, 33096, 31783, 32766, 32767] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=46910 max_load_reduction_pct=0.586316 moved_to_replicas=163162 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[32785, 33098, 33095, 32754, 33096, 31783, 32766, 32767] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=46910 max_load_reduction_pct=0.586316 moved_to_replicas=163162 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[32785, 33098, 33095, 32754, 33096, 31783, 32766, 32767] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=46910 max_load_reduction_pct=0.586316 moved_to_replicas=163162 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[32785, 33098, 33095, 32754, 33096, 31783, 32766, 32767] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=46910 max_load_reduction_pct=0.586316 moved_to_replicas=163162 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[39, 80008, 121, 32754, 65504, 18185, 32766, 32767] avg=32768.0000 max=80008 max_rank=1 min=39 min_rank=0 imbalance_ratio=2.441650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[32785, 33098, 33095, 32754, 33096, 31783, 32766, 32767] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=46910 max_load_reduction_pct=0.586316 moved_to_replicas=163162 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[39, 80008, 121, 32754, 65504, 18185, 32766, 32767] avg=32768.0000 max=80008 max_rank=1 min=39 min_rank=0 imbalance_ratio=2.441650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[18, 32780, 65473, 32766, 13, 32793, 32776, 65525] avg=32768.0000 max=65525 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[18, 32780, 65473, 32766, 13, 32793, 32776, 65525] avg=32768.0000 max=65525 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[18, 32780, 65473, 32766, 13, 32793, 32776, 65525] avg=32768.0000 max=65525 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[32785, 33098, 33095, 32754, 33096, 31783, 32766, 32767] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=46910 max_load_reduction_pct=0.586316 moved_to_replicas=163162 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[18, 32780, 65473, 32766, 13, 32793, 32776, 65525] avg=32768.0000 max=65525 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[32785, 33098, 33095, 32754, 33096, 31783, 32766, 32767] avg=32768.0000 max=33098 max_rank=1 min=31783 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=46910 max_load_reduction_pct=0.586316 moved_to_replicas=163162 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[18, 32780, 65473, 32766, 13, 32793, 32776, 65525] avg=32768.0000 max=65525 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[18, 32780, 65473, 32766, 13, 32793, 32776, 65525] avg=32768.0000 max=65525 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[32395, 32780, 33096, 32766, 32442, 32793, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=32395 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=129507 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[32395, 32780, 33096, 32766, 32442, 32793, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=32395 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=129507 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[32395, 32780, 33096, 32766, 32442, 32793, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=32395 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=129507 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[32395, 32780, 33096, 32766, 32442, 32793, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=32395 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=129507 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[32395, 32780, 33096, 32766, 32442, 32793, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=32395 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=129507 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[18, 32780, 65473, 32766, 13, 32793, 32776, 65525] avg=32768.0000 max=65525 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[32395, 32780, 33096, 32766, 32442, 32793, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=32395 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=129507 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[18, 32780, 65473, 32766, 13, 32793, 32776, 65525] avg=32768.0000 max=65525 max_rank=7 min=13 min_rank=4 imbalance_ratio=1.999664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[32765, 9, 32789, 32758, 65522, 32782, 65466, 53] avg=32768.0000 max=65522 max_rank=4 min=9 min_rank=1 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[32765, 9, 32789, 32758, 65522, 32782, 65466, 53] avg=32768.0000 max=65522 max_rank=4 min=9 min_rank=1 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[32395, 32780, 33096, 32766, 32442, 32793, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=32395 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=129507 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[32765, 9, 32789, 32758, 65522, 32782, 65466, 53] avg=32768.0000 max=65522 max_rank=4 min=9 min_rank=1 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[32765, 9, 32789, 32758, 65522, 32782, 65466, 53] avg=32768.0000 max=65522 max_rank=4 min=9 min_rank=1 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[32395, 32780, 33096, 32766, 32442, 32793, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=32395 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32429 max_load_reduction_pct=0.494910 moved_to_replicas=129507 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[32765, 9, 32789, 32758, 65522, 32782, 65466, 53] avg=32768.0000 max=65522 max_rank=4 min=9 min_rank=1 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[32765, 9, 32789, 32758, 65522, 32782, 65466, 53] avg=32768.0000 max=65522 max_rank=4 min=9 min_rank=1 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[32765, 32435, 32789, 32758, 33096, 32782, 33096, 32423] avg=32768.0000 max=33096 max_rank=4 min=32423 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129606 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[32765, 32435, 32789, 32758, 33096, 32782, 33096, 32423] avg=32768.0000 max=33096 max_rank=4 min=32423 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129606 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[32765, 32435, 32789, 32758, 33096, 32782, 33096, 32423] avg=32768.0000 max=33096 max_rank=4 min=32423 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129606 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[32765, 32435, 32789, 32758, 33096, 32782, 33096, 32423] avg=32768.0000 max=33096 max_rank=4 min=32423 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129606 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[32765, 32435, 32789, 32758, 33096, 32782, 33096, 32423] avg=32768.0000 max=33096 max_rank=4 min=32423 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129606 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[32765, 9, 32789, 32758, 65522, 32782, 65466, 53] avg=32768.0000 max=65522 max_rank=4 min=9 min_rank=1 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[32765, 9, 32789, 32758, 65522, 32782, 65466, 53] avg=32768.0000 max=65522 max_rank=4 min=9 min_rank=1 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[32765, 32435, 32789, 32758, 33096, 32782, 33096, 32423] avg=32768.0000 max=33096 max_rank=4 min=32423 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129606 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[65509, 32801, 0, 30, 32768, 22, 65501, 65513] avg=32768.0000 max=65513 max_rank=7 min=0 min_rank=2 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[65509, 32801, 0, 30, 32768, 22, 65501, 65513] avg=32768.0000 max=65513 max_rank=7 min=0 min_rank=2 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[32765, 32435, 32789, 32758, 33096, 32782, 33096, 32423] avg=32768.0000 max=33096 max_rank=4 min=32423 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129606 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[65509, 32801, 0, 30, 32768, 22, 65501, 65513] avg=32768.0000 max=65513 max_rank=7 min=0 min_rank=2 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[65509, 32801, 0, 30, 32768, 22, 65501, 65513] avg=32768.0000 max=65513 max_rank=7 min=0 min_rank=2 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[32765, 32435, 32789, 32758, 33096, 32782, 33096, 32423] avg=32768.0000 max=33096 max_rank=4 min=32423 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129606 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[65509, 32801, 0, 30, 32768, 22, 65501, 65513] avg=32768.0000 max=65513 max_rank=7 min=0 min_rank=2 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[65509, 32801, 0, 30, 32768, 22, 65501, 65513] avg=32768.0000 max=65513 max_rank=7 min=0 min_rank=2 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[33096, 32801, 32417, 32435, 32768, 32435, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32417 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=194453 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[33096, 32801, 32417, 32435, 32768, 32435, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32417 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=194453 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[33096, 32801, 32417, 32435, 32768, 32435, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32417 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=194453 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[33096, 32801, 32417, 32435, 32768, 32435, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32417 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=194453 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[33096, 32801, 32417, 32435, 32768, 32435, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32417 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=194453 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[65509, 32801, 0, 30, 32768, 22, 65501, 65513] avg=32768.0000 max=65513 max_rank=7 min=0 min_rank=2 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[33096, 32801, 32417, 32435, 32768, 32435, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32417 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=194453 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[65509, 32801, 0, 30, 32768, 22, 65501, 65513] avg=32768.0000 max=65513 max_rank=7 min=0 min_rank=2 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[33096, 32801, 32417, 32435, 32768, 32435, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32417 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=194453 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[33096, 32801, 32417, 32435, 32768, 32435, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32417 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=194453 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=32 total_expected_recv=1049493 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=251136 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=32 total_expected_recv=1042263 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=257664 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=32 total_expected_recv=1051070 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=249600 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=32 total_expected_recv=1042793 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=257024 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=32 total_expected_recv=1048313 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=251776 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=32 total_expected_recv=1052842 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=247424 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=32 total_expected_recv=1053099 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=248832 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=32 total_expected_recv=1048735 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=251648 peak_demand=33096 required_capacity=1.010010 + [2026-08-30 05:10:29] iteration 6/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 532.7 | throughput per GPU (TFLOP/s/GPU): 625.9 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.047608E+01 | loss scale: 1.0 | grad norm: 38.813 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=40960 expected_recv=65516 expected_overflow=0 exact_recv_per_rank=[33096, 32419, 33097, 33096, 33096, 31148, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32419, 33097, 33096, 33096, 31148, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33096, 32419, 33097, 33096, 33096, 31148, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32419, 33097, 33096, 33096, 31148, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=40960 expected_recv=64243 expected_overflow=0 exact_recv_per_rank=[33096, 32419, 33097, 33096, 33096, 31148, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=40960 expected_recv=63567 expected_overflow=0 exact_recv_per_rank=[33096, 32419, 33097, 33096, 33096, 31148, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32419, 33097, 33096, 33096, 31148, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=2 budget=40960 expected_recv=65516 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32419, 33097, 33096, 33096, 31148, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=2 budget=40960 expected_recv=64243 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=2 budget=40960 expected_recv=63567 actual_valid_recv=64128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=64128 slack_slots=17792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=40960 expected_recv=65576 expected_overflow=0 exact_recv_per_rank=[33095, 32774, 33096, 32786, 31478, 32724, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 32774, 33096, 32786, 31478, 32724, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=40960 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[33095, 32774, 33096, 32786, 31478, 32724, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=40960 expected_recv=65555 expected_overflow=0 exact_recv_per_rank=[33095, 32774, 33096, 32786, 31478, 32724, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=40960 expected_recv=65178 expected_overflow=0 exact_recv_per_rank=[33095, 32774, 33096, 32786, 31478, 32724, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=2 budget=40960 expected_recv=65576 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=2 budget=40960 expected_recv=66190 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=2 budget=40960 expected_recv=65555 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 32774, 33096, 32786, 31478, 32724, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=2 budget=40960 expected_recv=65178 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 32774, 33096, 32786, 31478, 32724, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=40960 expected_recv=63213 expected_overflow=0 exact_recv_per_rank=[33095, 32774, 33096, 32786, 31478, 32724, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=2 budget=40960 expected_recv=63213 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31806, 32768, 33095, 33096, 32445, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=40960 expected_recv=63644 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31806, 32768, 33095, 33096, 32445, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31806, 32768, 33095, 33096, 32445, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31806, 32768, 33095, 33096, 32445, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=2 budget=40960 expected_recv=63644 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31806, 32768, 33095, 33096, 32445, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65534 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31806, 32768, 33095, 33096, 32445, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65449 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31806, 32768, 33095, 33096, 32445, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31806, 32768, 33095, 33096, 32445, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65542 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65449 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=2 budget=40960 expected_recv=65544 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32768, 32444, 32840, 33096, 32035, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32768, 32444, 32840, 33096, 32035, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65543 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32768, 32444, 32840, 33096, 32035, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=40960 expected_recv=64878 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32768, 32444, 32840, 33096, 32035, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65542 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32768, 32444, 32840, 33096, 32035, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=2 budget=40960 expected_recv=64878 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=40960 expected_recv=64087 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32768, 32444, 32840, 33096, 32035, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32768, 32444, 32840, 33096, 32035, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65662 expected_overflow=0 exact_recv_per_rank=[33096, 32769, 32768, 32444, 32840, 33096, 32035, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=2 budget=40960 expected_recv=64087 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=2 budget=40960 expected_recv=65662 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32133, 33096, 33096, 32424, 33096, 32765, 33096, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32133, 33096, 33096, 32424, 33096, 32765, 33096, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=40960 expected_recv=64270 expected_overflow=0 exact_recv_per_rank=[32133, 33096, 33096, 32424, 33096, 32765, 33096, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=40960 expected_recv=64834 expected_overflow=0 exact_recv_per_rank=[32133, 33096, 33096, 32424, 33096, 32765, 33096, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=40960 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[32133, 33096, 33096, 32424, 33096, 32765, 33096, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=2 budget=40960 expected_recv=64270 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=2 budget=40960 expected_recv=64834 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32133, 33096, 33096, 32424, 33096, 32765, 33096, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=40960 expected_recv=64873 expected_overflow=0 exact_recv_per_rank=[32133, 33096, 33096, 32424, 33096, 32765, 33096, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=2 budget=40960 expected_recv=65543 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32133, 33096, 33096, 32424, 33096, 32765, 33096, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=2 budget=40960 expected_recv=64873 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=40960 expected_recv=64765 expected_overflow=0 exact_recv_per_rank=[33096, 32323, 32814, 32832, 32447, 33096, 32769, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65621 expected_overflow=0 exact_recv_per_rank=[33096, 32323, 32814, 32832, 32447, 33096, 32769, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32323, 32814, 32832, 32447, 33096, 32769, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65761 expected_overflow=0 exact_recv_per_rank=[33096, 32323, 32814, 32832, 32447, 33096, 32769, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=2 budget=40960 expected_recv=64765 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32323, 32814, 32832, 32447, 33096, 32769, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65621 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65761 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[33096, 32323, 32814, 32832, 32447, 33096, 32769, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[33096, 32323, 32814, 32832, 32447, 33096, 32769, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=40960 expected_recv=64678 expected_overflow=0 exact_recv_per_rank=[33096, 32323, 32814, 32832, 32447, 33096, 32769, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65543 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=2 budget=40960 expected_recv=65536 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=2 budget=40960 expected_recv=64678 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=40960 expected_recv=64935 expected_overflow=0 exact_recv_per_rank=[32433, 32456, 33096, 32760, 33096, 32772, 32760, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32433, 32456, 33096, 32760, 33096, 32772, 32760, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=40960 expected_recv=64861 expected_overflow=0 exact_recv_per_rank=[32433, 32456, 33096, 32760, 33096, 32772, 32760, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=2 budget=40960 expected_recv=64935 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[32433, 32456, 33096, 32760, 33096, 32772, 32760, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[32433, 32456, 33096, 32760, 33096, 32772, 32760, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=2 budget=40960 expected_recv=64861 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[32433, 32456, 33096, 32760, 33096, 32772, 32760, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65501 expected_overflow=0 exact_recv_per_rank=[32433, 32456, 33096, 32760, 33096, 32772, 32760, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65532 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65533 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32433, 32456, 33096, 32760, 33096, 32772, 32760, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65542 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=2 budget=40960 expected_recv=65501 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32432, 33096, 32650, 33096, 32766, 32903, 32763, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65063 expected_overflow=0 exact_recv_per_rank=[32432, 33096, 32650, 33096, 32766, 32903, 32763, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=40960 expected_recv=64879 expected_overflow=0 exact_recv_per_rank=[32432, 33096, 32650, 33096, 32766, 32903, 32763, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32432, 33096, 32650, 33096, 32766, 32903, 32763, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65063 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65999 expected_overflow=0 exact_recv_per_rank=[32432, 33096, 32650, 33096, 32766, 32903, 32763, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=2 budget=40960 expected_recv=64879 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=40960 expected_recv=64924 expected_overflow=0 exact_recv_per_rank=[32432, 33096, 32650, 33096, 32766, 32903, 32763, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65503 expected_overflow=0 exact_recv_per_rank=[32432, 33096, 32650, 33096, 32766, 32903, 32763, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[32432, 33096, 32650, 33096, 32766, 32903, 32763, 32438] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65999 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=2 budget=40960 expected_recv=64924 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65503 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=2 budget=40960 expected_recv=65536 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=40960 expected_recv=63766 expected_overflow=0 exact_recv_per_rank=[33096, 31895, 33096, 32776, 33026, 32453, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=2 budget=40960 expected_recv=63766 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31895, 33096, 32776, 33026, 32453, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31895, 33096, 32776, 33026, 32453, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=40960 expected_recv=65559 expected_overflow=0 exact_recv_per_rank=[33096, 31895, 33096, 32776, 33026, 32453, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=40960 expected_recv=64942 expected_overflow=0 exact_recv_per_rank=[33096, 31895, 33096, 32776, 33026, 32453, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31895, 33096, 32776, 33026, 32453, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=40960 expected_recv=65323 expected_overflow=0 exact_recv_per_rank=[33096, 31895, 33096, 32776, 33026, 32453, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=2 budget=40960 expected_recv=65559 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=40960 expected_recv=66122 expected_overflow=0 exact_recv_per_rank=[33096, 31895, 33096, 32776, 33026, 32453, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=2 budget=40960 expected_recv=64942 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=2 budget=40960 expected_recv=65323 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=2 budget=40960 expected_recv=66122 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31678, 32799, 32847, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31678, 32799, 32847, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31678, 32799, 32847, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31678, 32799, 32847, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=40960 expected_recv=65624 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31678, 32799, 32847, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=40960 expected_recv=64160 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31678, 32799, 32847, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=40960 expected_recv=65624 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31678, 32799, 32847, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=2 budget=40960 expected_recv=65624 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=40960 expected_recv=64112 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31678, 32799, 32847, 32436] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=2 budget=40960 expected_recv=64160 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=2 budget=40960 expected_recv=65624 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=40960 expected_recv=64169 expected_overflow=0 exact_recv_per_rank=[32775, 32094, 32771, 32767, 33096, 33096, 32776, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=2 budget=40960 expected_recv=64112 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=2 budget=40960 expected_recv=64169 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[32775, 32094, 32771, 32767, 33096, 33096, 32776, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65552 expected_overflow=0 exact_recv_per_rank=[32775, 32094, 32771, 32767, 33096, 33096, 32776, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65536 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65552 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[32775, 32094, 32771, 32767, 33096, 33096, 32776, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32775, 32094, 32771, 32767, 33096, 33096, 32776, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[32775, 32094, 32771, 32767, 33096, 33096, 32776, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65564 expected_overflow=0 exact_recv_per_rank=[32775, 32094, 32771, 32767, 33096, 33096, 32776, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65537 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32775, 32094, 32771, 32767, 33096, 33096, 32776, 32769] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65546 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32773, 32721, 31822, 32773, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=2 budget=40960 expected_recv=65564 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32773, 32721, 31822, 32773, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65539 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32773, 32721, 31822, 32773, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65547 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32773, 32721, 31822, 32773, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65434 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32773, 32721, 31822, 32773, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65547 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32773, 32721, 31822, 32773, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65434 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32773, 32721, 31822, 32773, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=40960 expected_recv=63659 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32773, 32721, 31822, 32773, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32767, 32775, 33096, 32784, 32415, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=2 budget=40960 expected_recv=65533 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=2 budget=40960 expected_recv=63659 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32767, 32775, 33096, 32784, 32415, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65541 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=40960 expected_recv=64892 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32767, 32775, 33096, 32784, 32415, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65585 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32767, 32775, 33096, 32784, 32415, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=2 budget=40960 expected_recv=64892 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65555 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32767, 32775, 33096, 32784, 32415, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65585 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65512 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32767, 32775, 33096, 32784, 32415, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65555 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=2 budget=40960 expected_recv=65512 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=40960 expected_recv=64819 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32767, 32775, 33096, 32784, 32415, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32443, 33096, 32767, 32775, 33096, 32784, 32415, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[32441, 32776, 33096, 32764, 32450, 32776, 32745, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=12 calls=2 budget=40960 expected_recv=64819 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32441, 32776, 33096, 32764, 32450, 32776, 32745, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65533 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=40960 expected_recv=64894 expected_overflow=0 exact_recv_per_rank=[32441, 32776, 33096, 32764, 32450, 32776, 32745, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65568 expected_overflow=0 exact_recv_per_rank=[32441, 32776, 33096, 32764, 32450, 32776, 32745, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=13 calls=2 budget=40960 expected_recv=64894 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[32441, 32776, 33096, 32764, 32450, 32776, 32745, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65568 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32441, 32776, 33096, 32764, 32450, 32776, 32745, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65527 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65479 expected_overflow=0 exact_recv_per_rank=[32441, 32776, 33096, 32764, 32450, 32776, 32745, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=40960 expected_recv=64903 expected_overflow=0 exact_recv_per_rank=[32441, 32776, 33096, 32764, 32450, 32776, 32745, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65504 expected_overflow=0 exact_recv_per_rank=[32774, 32755, 32589, 32776, 32423, 33096, 33096, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=2 budget=40960 expected_recv=65479 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=13 calls=2 budget=40960 expected_recv=64903 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65275 expected_overflow=0 exact_recv_per_rank=[32774, 32755, 32589, 32776, 32423, 33096, 33096, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65504 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[32774, 32755, 32589, 32776, 32423, 33096, 33096, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65275 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32774, 32755, 32589, 32776, 32423, 33096, 33096, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65550 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65175 expected_overflow=0 exact_recv_per_rank=[32774, 32755, 32589, 32776, 32423, 33096, 33096, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[32774, 32755, 32589, 32776, 32423, 33096, 33096, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65175 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=14 calls=2 budget=40960 expected_recv=65550 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32774, 32755, 32589, 32776, 32423, 33096, 33096, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=40960 expected_recv=64850 expected_overflow=0 exact_recv_per_rank=[32774, 32755, 32589, 32776, 32423, 33096, 33096, 32635] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65134 expected_overflow=0 exact_recv_per_rank=[33096, 32537, 32725, 32440, 32758, 32396, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=14 calls=2 budget=40960 expected_recv=64850 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65398 expected_overflow=0 exact_recv_per_rank=[33096, 32537, 32725, 32440, 32758, 32396, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65134 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32537, 32725, 32440, 32758, 32396, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=40960 expected_recv=64791 expected_overflow=0 exact_recv_per_rank=[33096, 32537, 32725, 32440, 32758, 32396, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65398 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=2 budget=40960 expected_recv=64791 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32537, 32725, 32440, 32758, 32396, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=40960 expected_recv=64877 expected_overflow=0 exact_recv_per_rank=[33096, 32537, 32725, 32440, 32758, 32396, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32537, 32725, 32440, 32758, 32396, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=15 calls=2 budget=40960 expected_recv=64877 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65512 expected_overflow=0 exact_recv_per_rank=[33096, 32537, 32725, 32440, 32758, 32396, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=15 calls=2 budget=40960 expected_recv=65512 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[7104, 17688, 39439, 56547, 64858, 18517, 37956, 20035] avg=32768.0000 max=64858 max_rank=4 min=7104 min_rank=0 imbalance_ratio=1.979309 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[7104, 17688, 39439, 56547, 64858, 18517, 37956, 20035] avg=32768.0000 max=64858 max_rank=4 min=7104 min_rank=0 imbalance_ratio=1.979309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[7104, 17688, 39439, 56547, 64858, 18517, 37956, 20035] avg=32768.0000 max=64858 max_rank=4 min=7104 min_rank=0 imbalance_ratio=1.979309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[7104, 17688, 39439, 56547, 64858, 18517, 37956, 20035] avg=32768.0000 max=64858 max_rank=4 min=7104 min_rank=0 imbalance_ratio=1.979309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[33096, 32419, 33097, 33096, 33096, 31148, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31148 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31761 max_load_reduction_pct=0.489701 moved_to_replicas=132025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[7104, 17688, 39439, 56547, 64858, 18517, 37956, 20035] avg=32768.0000 max=64858 max_rank=4 min=7104 min_rank=0 imbalance_ratio=1.979309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[7104, 17688, 39439, 56547, 64858, 18517, 37956, 20035] avg=32768.0000 max=64858 max_rank=4 min=7104 min_rank=0 imbalance_ratio=1.979309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[7104, 17688, 39439, 56547, 64858, 18517, 37956, 20035] avg=32768.0000 max=64858 max_rank=4 min=7104 min_rank=0 imbalance_ratio=1.979309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[33096, 32419, 33097, 33096, 33096, 31148, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31148 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31761 max_load_reduction_pct=0.489701 moved_to_replicas=132025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[33096, 32419, 33097, 33096, 33096, 31148, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31148 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31761 max_load_reduction_pct=0.489701 moved_to_replicas=132025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[33096, 32419, 33097, 33096, 33096, 31148, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31148 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31761 max_load_reduction_pct=0.489701 moved_to_replicas=132025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[7104, 17688, 39439, 56547, 64858, 18517, 37956, 20035] avg=32768.0000 max=64858 max_rank=4 min=7104 min_rank=0 imbalance_ratio=1.979309 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[33096, 32419, 33097, 33096, 33096, 31148, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31148 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31761 max_load_reduction_pct=0.489701 moved_to_replicas=132025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[33096, 32419, 33097, 33096, 33096, 31148, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31148 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31761 max_load_reduction_pct=0.489701 moved_to_replicas=132025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[39183, 40, 7780, 32786, 4495, 32724, 79624, 65512] avg=32768.0000 max=79624 max_rank=6 min=40 min_rank=1 imbalance_ratio=2.429932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[33096, 32419, 33097, 33096, 33096, 31148, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31148 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31761 max_load_reduction_pct=0.489701 moved_to_replicas=132025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[33096, 32419, 33097, 33096, 33096, 31148, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31148 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=31761 max_load_reduction_pct=0.489701 moved_to_replicas=132025 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[39183, 40, 7780, 32786, 4495, 32724, 79624, 65512] avg=32768.0000 max=79624 max_rank=6 min=40 min_rank=1 imbalance_ratio=2.429932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[39183, 40, 7780, 32786, 4495, 32724, 79624, 65512] avg=32768.0000 max=79624 max_rank=6 min=40 min_rank=1 imbalance_ratio=2.429932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[39183, 40, 7780, 32786, 4495, 32724, 79624, 65512] avg=32768.0000 max=79624 max_rank=6 min=40 min_rank=1 imbalance_ratio=2.429932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[33095, 32774, 33096, 32786, 31478, 32724, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31478 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46528 max_load_reduction_pct=0.584346 moved_to_replicas=169937 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[39183, 40, 7780, 32786, 4495, 32724, 79624, 65512] avg=32768.0000 max=79624 max_rank=6 min=40 min_rank=1 imbalance_ratio=2.429932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[39183, 40, 7780, 32786, 4495, 32724, 79624, 65512] avg=32768.0000 max=79624 max_rank=6 min=40 min_rank=1 imbalance_ratio=2.429932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[39183, 40, 7780, 32786, 4495, 32724, 79624, 65512] avg=32768.0000 max=79624 max_rank=6 min=40 min_rank=1 imbalance_ratio=2.429932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[33095, 32774, 33096, 32786, 31478, 32724, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31478 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46528 max_load_reduction_pct=0.584346 moved_to_replicas=169937 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[33095, 32774, 33096, 32786, 31478, 32724, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31478 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46528 max_load_reduction_pct=0.584346 moved_to_replicas=169937 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[33095, 32774, 33096, 32786, 31478, 32724, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31478 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46528 max_load_reduction_pct=0.584346 moved_to_replicas=169937 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[39183, 40, 7780, 32786, 4495, 32724, 79624, 65512] avg=32768.0000 max=79624 max_rank=6 min=40 min_rank=1 imbalance_ratio=2.429932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[33095, 32774, 33096, 32786, 31478, 32724, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31478 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46528 max_load_reduction_pct=0.584346 moved_to_replicas=169937 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[33095, 32774, 33096, 32786, 31478, 32724, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31478 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46528 max_load_reduction_pct=0.584346 moved_to_replicas=169937 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[65528, 77066, 21239, 32768, 10, 65517, 13, 3] avg=32768.0000 max=77066 max_rank=1 min=3 min_rank=7 imbalance_ratio=2.351868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[33095, 32774, 33096, 32786, 31478, 32724, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31478 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46528 max_load_reduction_pct=0.584346 moved_to_replicas=169937 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[33095, 32774, 33096, 32786, 31478, 32724, 33095, 33096] avg=32768.0000 max=33096 max_rank=2 min=31478 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46528 max_load_reduction_pct=0.584346 moved_to_replicas=169937 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[65528, 77066, 21239, 32768, 10, 65517, 13, 3] avg=32768.0000 max=77066 max_rank=1 min=3 min_rank=7 imbalance_ratio=2.351868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33096, 31806, 32768, 33095, 33096, 32445, 32742] avg=32768.0000 max=33096 max_rank=0 min=31806 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=43970 max_load_reduction_pct=0.570550 moved_to_replicas=220331 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[65528, 77066, 21239, 32768, 10, 65517, 13, 3] avg=32768.0000 max=77066 max_rank=1 min=3 min_rank=7 imbalance_ratio=2.351868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[65528, 77066, 21239, 32768, 10, 65517, 13, 3] avg=32768.0000 max=77066 max_rank=1 min=3 min_rank=7 imbalance_ratio=2.351868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[65528, 77066, 21239, 32768, 10, 65517, 13, 3] avg=32768.0000 max=77066 max_rank=1 min=3 min_rank=7 imbalance_ratio=2.351868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[65528, 77066, 21239, 32768, 10, 65517, 13, 3] avg=32768.0000 max=77066 max_rank=1 min=3 min_rank=7 imbalance_ratio=2.351868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[65528, 77066, 21239, 32768, 10, 65517, 13, 3] avg=32768.0000 max=77066 max_rank=1 min=3 min_rank=7 imbalance_ratio=2.351868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33096, 31806, 32768, 33095, 33096, 32445, 32742] avg=32768.0000 max=33096 max_rank=0 min=31806 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=43970 max_load_reduction_pct=0.570550 moved_to_replicas=220331 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33096, 31806, 32768, 33095, 33096, 32445, 32742] avg=32768.0000 max=33096 max_rank=0 min=31806 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=43970 max_load_reduction_pct=0.570550 moved_to_replicas=220331 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[65528, 77066, 21239, 32768, 10, 65517, 13, 3] avg=32768.0000 max=77066 max_rank=1 min=3 min_rank=7 imbalance_ratio=2.351868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33096, 31806, 32768, 33095, 33096, 32445, 32742] avg=32768.0000 max=33096 max_rank=0 min=31806 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=43970 max_load_reduction_pct=0.570550 moved_to_replicas=220331 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33096, 31806, 32768, 33095, 33096, 32445, 32742] avg=32768.0000 max=33096 max_rank=0 min=31806 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=43970 max_load_reduction_pct=0.570550 moved_to_replicas=220331 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[65531, 32769, 32768, 9, 32840, 34390, 738, 63099] avg=32768.0000 max=65531 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33096, 31806, 32768, 33095, 33096, 32445, 32742] avg=32768.0000 max=33096 max_rank=0 min=31806 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=43970 max_load_reduction_pct=0.570550 moved_to_replicas=220331 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33096, 31806, 32768, 33095, 33096, 32445, 32742] avg=32768.0000 max=33096 max_rank=0 min=31806 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=43970 max_load_reduction_pct=0.570550 moved_to_replicas=220331 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[33096, 33096, 31806, 32768, 33095, 33096, 32445, 32742] avg=32768.0000 max=33096 max_rank=0 min=31806 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=43970 max_load_reduction_pct=0.570550 moved_to_replicas=220331 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[65531, 32769, 32768, 9, 32840, 34390, 738, 63099] avg=32768.0000 max=65531 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[65531, 32769, 32768, 9, 32840, 34390, 738, 63099] avg=32768.0000 max=65531 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[33096, 32769, 32768, 32444, 32840, 33096, 32035, 33096] avg=32768.0000 max=33096 max_rank=0 min=32035 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=127557 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[65531, 32769, 32768, 9, 32840, 34390, 738, 63099] avg=32768.0000 max=65531 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[65531, 32769, 32768, 9, 32840, 34390, 738, 63099] avg=32768.0000 max=65531 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[65531, 32769, 32768, 9, 32840, 34390, 738, 63099] avg=32768.0000 max=65531 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[65531, 32769, 32768, 9, 32840, 34390, 738, 63099] avg=32768.0000 max=65531 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[33096, 32769, 32768, 32444, 32840, 33096, 32035, 33096] avg=32768.0000 max=33096 max_rank=0 min=32035 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=127557 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[33096, 32769, 32768, 32444, 32840, 33096, 32035, 33096] avg=32768.0000 max=33096 max_rank=0 min=32035 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=127557 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[65531, 32769, 32768, 9, 32840, 34390, 738, 63099] avg=32768.0000 max=65531 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[33096, 32769, 32768, 32444, 32840, 33096, 32035, 33096] avg=32768.0000 max=33096 max_rank=0 min=32035 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=127557 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[33096, 32769, 32768, 32444, 32840, 33096, 32035, 33096] avg=32768.0000 max=33096 max_rank=0 min=32035 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=127557 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[29, 64992, 33304, 14, 65529, 32765, 65506, 5] avg=32768.0000 max=65529 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[33096, 32769, 32768, 32444, 32840, 33096, 32035, 33096] avg=32768.0000 max=33096 max_rank=0 min=32035 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=127557 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[33096, 32769, 32768, 32444, 32840, 33096, 32035, 33096] avg=32768.0000 max=33096 max_rank=0 min=32035 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=127557 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[33096, 32769, 32768, 32444, 32840, 33096, 32035, 33096] avg=32768.0000 max=33096 max_rank=0 min=32035 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=127557 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[29, 64992, 33304, 14, 65529, 32765, 65506, 5] avg=32768.0000 max=65529 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[29, 64992, 33304, 14, 65529, 32765, 65506, 5] avg=32768.0000 max=65529 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[32133, 33096, 33096, 32424, 33096, 32765, 33096, 32438] avg=32768.0000 max=33096 max_rank=1 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=193830 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[29, 64992, 33304, 14, 65529, 32765, 65506, 5] avg=32768.0000 max=65529 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[29, 64992, 33304, 14, 65529, 32765, 65506, 5] avg=32768.0000 max=65529 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[29, 64992, 33304, 14, 65529, 32765, 65506, 5] avg=32768.0000 max=65529 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[29, 64992, 33304, 14, 65529, 32765, 65506, 5] avg=32768.0000 max=65529 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[32133, 33096, 33096, 32424, 33096, 32765, 33096, 32438] avg=32768.0000 max=33096 max_rank=1 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=193830 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[32133, 33096, 33096, 32424, 33096, 32765, 33096, 32438] avg=32768.0000 max=33096 max_rank=1 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=193830 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[29, 64992, 33304, 14, 65529, 32765, 65506, 5] avg=32768.0000 max=65529 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[32133, 33096, 33096, 32424, 33096, 32765, 33096, 32438] avg=32768.0000 max=33096 max_rank=1 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=193830 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[32133, 33096, 33096, 32424, 33096, 32765, 33096, 32438] avg=32768.0000 max=33096 max_rank=1 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=193830 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[98168, 12, 32814, 32832, 13, 65530, 8, 32767] avg=32768.0000 max=98168 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[32133, 33096, 33096, 32424, 33096, 32765, 33096, 32438] avg=32768.0000 max=33096 max_rank=1 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=193830 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[32133, 33096, 33096, 32424, 33096, 32765, 33096, 32438] avg=32768.0000 max=33096 max_rank=1 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=193830 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[32133, 33096, 33096, 32424, 33096, 32765, 33096, 32438] avg=32768.0000 max=33096 max_rank=1 min=32133 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=193830 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[98168, 12, 32814, 32832, 13, 65530, 8, 32767] avg=32768.0000 max=98168 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[98168, 12, 32814, 32832, 13, 65530, 8, 32767] avg=32768.0000 max=98168 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32323, 32814, 32832, 32447, 33096, 32769, 32767] avg=32768.0000 max=33096 max_rank=0 min=32323 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65072 max_load_reduction_pct=0.662864 moved_to_replicas=194893 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[98168, 12, 32814, 32832, 13, 65530, 8, 32767] avg=32768.0000 max=98168 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[98168, 12, 32814, 32832, 13, 65530, 8, 32767] avg=32768.0000 max=98168 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[98168, 12, 32814, 32832, 13, 65530, 8, 32767] avg=32768.0000 max=98168 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[98168, 12, 32814, 32832, 13, 65530, 8, 32767] avg=32768.0000 max=98168 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32323, 32814, 32832, 32447, 33096, 32769, 32767] avg=32768.0000 max=33096 max_rank=0 min=32323 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65072 max_load_reduction_pct=0.662864 moved_to_replicas=194893 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32323, 32814, 32832, 32447, 33096, 32769, 32767] avg=32768.0000 max=33096 max_rank=0 min=32323 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65072 max_load_reduction_pct=0.662864 moved_to_replicas=194893 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[98168, 12, 32814, 32832, 13, 65530, 8, 32767] avg=32768.0000 max=98168 max_rank=0 min=8 min_rank=6 imbalance_ratio=2.995850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32323, 32814, 32832, 32447, 33096, 32769, 32767] avg=32768.0000 max=33096 max_rank=0 min=32323 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65072 max_load_reduction_pct=0.662864 moved_to_replicas=194893 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32323, 32814, 32832, 32447, 33096, 32769, 32767] avg=32768.0000 max=33096 max_rank=0 min=32323 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65072 max_load_reduction_pct=0.662864 moved_to_replicas=194893 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[7, 30, 65522, 32760, 65522, 32772, 32760, 32771] avg=32768.0000 max=65522 max_rank=2 min=7 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32323, 32814, 32832, 32447, 33096, 32769, 32767] avg=32768.0000 max=33096 max_rank=0 min=32323 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65072 max_load_reduction_pct=0.662864 moved_to_replicas=194893 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32323, 32814, 32832, 32447, 33096, 32769, 32767] avg=32768.0000 max=33096 max_rank=0 min=32323 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65072 max_load_reduction_pct=0.662864 moved_to_replicas=194893 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[33096, 32323, 32814, 32832, 32447, 33096, 32769, 32767] avg=32768.0000 max=33096 max_rank=0 min=32323 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65072 max_load_reduction_pct=0.662864 moved_to_replicas=194893 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[7, 30, 65522, 32760, 65522, 32772, 32760, 32771] avg=32768.0000 max=65522 max_rank=2 min=7 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[32433, 32456, 33096, 32760, 33096, 32772, 32760, 32771] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129682 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[7, 30, 65522, 32760, 65522, 32772, 32760, 32771] avg=32768.0000 max=65522 max_rank=2 min=7 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[7, 30, 65522, 32760, 65522, 32772, 32760, 32771] avg=32768.0000 max=65522 max_rank=2 min=7 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[7, 30, 65522, 32760, 65522, 32772, 32760, 32771] avg=32768.0000 max=65522 max_rank=2 min=7 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[7, 30, 65522, 32760, 65522, 32772, 32760, 32771] avg=32768.0000 max=65522 max_rank=2 min=7 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[7, 30, 65522, 32760, 65522, 32772, 32760, 32771] avg=32768.0000 max=65522 max_rank=2 min=7 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[32433, 32456, 33096, 32760, 33096, 32772, 32760, 32771] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129682 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[32433, 32456, 33096, 32760, 33096, 32772, 32760, 32771] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129682 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[7, 30, 65522, 32760, 65522, 32772, 32760, 32771] avg=32768.0000 max=65522 max_rank=2 min=7 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[32433, 32456, 33096, 32760, 33096, 32772, 32760, 32771] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129682 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[32433, 32456, 33096, 32760, 33096, 32772, 32760, 32771] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129682 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[18, 65526, 32650, 65510, 32766, 32903, 32763, 8] avg=32768.0000 max=65526 max_rank=1 min=8 min_rank=7 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[32433, 32456, 33096, 32760, 33096, 32772, 32760, 32771] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129682 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[32433, 32456, 33096, 32760, 33096, 32772, 32760, 32771] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129682 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[32433, 32456, 33096, 32760, 33096, 32772, 32760, 32771] avg=32768.0000 max=33096 max_rank=2 min=32433 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=129682 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[18, 65526, 32650, 65510, 32766, 32903, 32763, 8] avg=32768.0000 max=65526 max_rank=1 min=8 min_rank=7 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[32432, 33096, 32650, 33096, 32766, 32903, 32763, 32438] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=129733 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[18, 65526, 32650, 65510, 32766, 32903, 32763, 8] avg=32768.0000 max=65526 max_rank=1 min=8 min_rank=7 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[18, 65526, 32650, 65510, 32766, 32903, 32763, 8] avg=32768.0000 max=65526 max_rank=1 min=8 min_rank=7 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[18, 65526, 32650, 65510, 32766, 32903, 32763, 8] avg=32768.0000 max=65526 max_rank=1 min=8 min_rank=7 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[32432, 33096, 32650, 33096, 32766, 32903, 32763, 32438] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=129733 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[18, 65526, 32650, 65510, 32766, 32903, 32763, 8] avg=32768.0000 max=65526 max_rank=1 min=8 min_rank=7 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[18, 65526, 32650, 65510, 32766, 32903, 32763, 8] avg=32768.0000 max=65526 max_rank=1 min=8 min_rank=7 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[32432, 33096, 32650, 33096, 32766, 32903, 32763, 32438] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=129733 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[18, 65526, 32650, 65510, 32766, 32903, 32763, 8] avg=32768.0000 max=65526 max_rank=1 min=8 min_rank=7 imbalance_ratio=1.999695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[32432, 33096, 32650, 33096, 32766, 32903, 32763, 32438] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=129733 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[32432, 33096, 32650, 33096, 32766, 32903, 32763, 32438] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=129733 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[65514, 31895, 66446, 17, 20, 35, 32706, 65511] avg=32768.0000 max=66446 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.027771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[32432, 33096, 32650, 33096, 32766, 32903, 32763, 32438] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=129733 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[32432, 33096, 32650, 33096, 32766, 32903, 32763, 32438] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=129733 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[32432, 33096, 32650, 33096, 32766, 32903, 32763, 32438] avg=32768.0000 max=33096 max_rank=1 min=32432 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32430 max_load_reduction_pct=0.494918 moved_to_replicas=129733 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[65514, 31895, 66446, 17, 20, 35, 32706, 65511] avg=32768.0000 max=66446 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.027771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[65514, 31895, 66446, 17, 20, 35, 32706, 65511] avg=32768.0000 max=66446 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.027771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[33096, 31895, 33096, 32776, 33026, 32453, 32706, 33096] avg=32768.0000 max=33096 max_rank=0 min=31895 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33350 max_load_reduction_pct=0.501911 moved_to_replicas=196640 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[65514, 31895, 66446, 17, 20, 35, 32706, 65511] avg=32768.0000 max=66446 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.027771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[65514, 31895, 66446, 17, 20, 35, 32706, 65511] avg=32768.0000 max=66446 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.027771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[33096, 31895, 33096, 32776, 33026, 32453, 32706, 33096] avg=32768.0000 max=33096 max_rank=0 min=31895 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33350 max_load_reduction_pct=0.501911 moved_to_replicas=196640 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[65514, 31895, 66446, 17, 20, 35, 32706, 65511] avg=32768.0000 max=66446 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.027771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[65514, 31895, 66446, 17, 20, 35, 32706, 65511] avg=32768.0000 max=66446 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.027771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[65514, 31895, 66446, 17, 20, 35, 32706, 65511] avg=32768.0000 max=66446 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.027771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[33096, 31895, 33096, 32776, 33026, 32453, 32706, 33096] avg=32768.0000 max=33096 max_rank=0 min=31895 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33350 max_load_reduction_pct=0.501911 moved_to_replicas=196640 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[33096, 31895, 33096, 32776, 33026, 32453, 32706, 33096] avg=32768.0000 max=33096 max_rank=0 min=31895 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33350 max_load_reduction_pct=0.501911 moved_to_replicas=196640 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[33096, 31895, 33096, 32776, 33026, 32453, 32706, 33096] avg=32768.0000 max=33096 max_rank=0 min=31895 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33350 max_load_reduction_pct=0.501911 moved_to_replicas=196640 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[33399, 65522, 63227, 34329, 11, 32799, 32847, 10] avg=32768.0000 max=65522 max_rank=1 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[33096, 31895, 33096, 32776, 33026, 32453, 32706, 33096] avg=32768.0000 max=33096 max_rank=0 min=31895 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33350 max_load_reduction_pct=0.501911 moved_to_replicas=196640 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[33096, 31895, 33096, 32776, 33026, 32453, 32706, 33096] avg=32768.0000 max=33096 max_rank=0 min=31895 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33350 max_load_reduction_pct=0.501911 moved_to_replicas=196640 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[33399, 65522, 63227, 34329, 11, 32799, 32847, 10] avg=32768.0000 max=65522 max_rank=1 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[33096, 31895, 33096, 32776, 33026, 32453, 32706, 33096] avg=32768.0000 max=33096 max_rank=0 min=31895 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=33350 max_load_reduction_pct=0.501911 moved_to_replicas=196640 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31678, 32799, 32847, 32436] avg=32768.0000 max=33096 max_rank=0 min=31678 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128210 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[33399, 65522, 63227, 34329, 11, 32799, 32847, 10] avg=32768.0000 max=65522 max_rank=1 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[33399, 65522, 63227, 34329, 11, 32799, 32847, 10] avg=32768.0000 max=65522 max_rank=1 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[33399, 65522, 63227, 34329, 11, 32799, 32847, 10] avg=32768.0000 max=65522 max_rank=1 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31678, 32799, 32847, 32436] avg=32768.0000 max=33096 max_rank=0 min=31678 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128210 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[33399, 65522, 63227, 34329, 11, 32799, 32847, 10] avg=32768.0000 max=65522 max_rank=1 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[33399, 65522, 63227, 34329, 11, 32799, 32847, 10] avg=32768.0000 max=65522 max_rank=1 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[33399, 65522, 63227, 34329, 11, 32799, 32847, 10] avg=32768.0000 max=65522 max_rank=1 min=10 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31678, 32799, 32847, 32436] avg=32768.0000 max=33096 max_rank=0 min=31678 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128210 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31678, 32799, 32847, 32436] avg=32768.0000 max=33096 max_rank=0 min=31678 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128210 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31678, 32799, 32847, 32436] avg=32768.0000 max=33096 max_rank=0 min=31678 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128210 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[32775, 6, 32771, 32767, 34921, 63359, 32776, 32769] avg=32768.0000 max=63359 max_rank=5 min=6 min_rank=1 imbalance_ratio=1.933563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31678, 32799, 32847, 32436] avg=32768.0000 max=33096 max_rank=0 min=31678 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128210 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31678, 32799, 32847, 32436] avg=32768.0000 max=33096 max_rank=0 min=31678 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128210 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31678, 32799, 32847, 32436] avg=32768.0000 max=33096 max_rank=0 min=31678 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128210 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[32775, 6, 32771, 32767, 34921, 63359, 32776, 32769] avg=32768.0000 max=63359 max_rank=5 min=6 min_rank=1 imbalance_ratio=1.933563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[32775, 32094, 32771, 32767, 33096, 33096, 32776, 32769] avg=32768.0000 max=33096 max_rank=4 min=32094 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30263 max_load_reduction_pct=0.477643 moved_to_replicas=64157 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[32775, 6, 32771, 32767, 34921, 63359, 32776, 32769] avg=32768.0000 max=63359 max_rank=5 min=6 min_rank=1 imbalance_ratio=1.933563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[32775, 6, 32771, 32767, 34921, 63359, 32776, 32769] avg=32768.0000 max=63359 max_rank=5 min=6 min_rank=1 imbalance_ratio=1.933563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[32775, 6, 32771, 32767, 34921, 63359, 32776, 32769] avg=32768.0000 max=63359 max_rank=5 min=6 min_rank=1 imbalance_ratio=1.933563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[32775, 32094, 32771, 32767, 33096, 33096, 32776, 32769] avg=32768.0000 max=33096 max_rank=4 min=32094 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30263 max_load_reduction_pct=0.477643 moved_to_replicas=64157 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[32775, 6, 32771, 32767, 34921, 63359, 32776, 32769] avg=32768.0000 max=63359 max_rank=5 min=6 min_rank=1 imbalance_ratio=1.933563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[32775, 6, 32771, 32767, 34921, 63359, 32776, 32769] avg=32768.0000 max=63359 max_rank=5 min=6 min_rank=1 imbalance_ratio=1.933563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[32775, 6, 32771, 32767, 34921, 63359, 32776, 32769] avg=32768.0000 max=63359 max_rank=5 min=6 min_rank=1 imbalance_ratio=1.933563 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[32775, 32094, 32771, 32767, 33096, 33096, 32776, 32769] avg=32768.0000 max=33096 max_rank=4 min=32094 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30263 max_load_reduction_pct=0.477643 moved_to_replicas=64157 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[32775, 32094, 32771, 32767, 33096, 33096, 32776, 32769] avg=32768.0000 max=33096 max_rank=4 min=32094 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30263 max_load_reduction_pct=0.477643 moved_to_replicas=64157 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[32775, 32094, 32771, 32767, 33096, 33096, 32776, 32769] avg=32768.0000 max=33096 max_rank=4 min=32094 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30263 max_load_reduction_pct=0.477643 moved_to_replicas=64157 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[32775, 32094, 32771, 32767, 33096, 33096, 32776, 32769] avg=32768.0000 max=33096 max_rank=4 min=32094 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30263 max_load_reduction_pct=0.477643 moved_to_replicas=64157 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[32775, 32094, 32771, 32767, 33096, 33096, 32776, 32769] avg=32768.0000 max=33096 max_rank=4 min=32094 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30263 max_load_reduction_pct=0.477643 moved_to_replicas=64157 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[65521, 97116, 32773, 16, 37, 32773, 32767, 1141] avg=32768.0000 max=97116 max_rank=1 min=16 min_rank=3 imbalance_ratio=2.963745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[32775, 32094, 32771, 32767, 33096, 33096, 32776, 32769] avg=32768.0000 max=33096 max_rank=4 min=32094 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=30263 max_load_reduction_pct=0.477643 moved_to_replicas=64157 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[65521, 97116, 32773, 16, 37, 32773, 32767, 1141] avg=32768.0000 max=97116 max_rank=1 min=16 min_rank=3 imbalance_ratio=2.963745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[65521, 97116, 32773, 16, 37, 32773, 32767, 1141] avg=32768.0000 max=97116 max_rank=1 min=16 min_rank=3 imbalance_ratio=2.963745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[65521, 97116, 32773, 16, 37, 32773, 32767, 1141] avg=32768.0000 max=97116 max_rank=1 min=16 min_rank=3 imbalance_ratio=2.963745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 32773, 32721, 31822, 32773, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31822 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64020 max_load_reduction_pct=0.659212 moved_to_replicas=190978 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 32773, 32721, 31822, 32773, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31822 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64020 max_load_reduction_pct=0.659212 moved_to_replicas=190978 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[65521, 97116, 32773, 16, 37, 32773, 32767, 1141] avg=32768.0000 max=97116 max_rank=1 min=16 min_rank=3 imbalance_ratio=2.963745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[65521, 97116, 32773, 16, 37, 32773, 32767, 1141] avg=32768.0000 max=97116 max_rank=1 min=16 min_rank=3 imbalance_ratio=2.963745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[65521, 97116, 32773, 16, 37, 32773, 32767, 1141] avg=32768.0000 max=97116 max_rank=1 min=16 min_rank=3 imbalance_ratio=2.963745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[65521, 97116, 32773, 16, 37, 32773, 32767, 1141] avg=32768.0000 max=97116 max_rank=1 min=16 min_rank=3 imbalance_ratio=2.963745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 32773, 32721, 31822, 32773, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31822 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64020 max_load_reduction_pct=0.659212 moved_to_replicas=190978 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 32773, 32721, 31822, 32773, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31822 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64020 max_load_reduction_pct=0.659212 moved_to_replicas=190978 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 32773, 32721, 31822, 32773, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31822 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64020 max_load_reduction_pct=0.659212 moved_to_replicas=190978 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 32773, 32721, 31822, 32773, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31822 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64020 max_load_reduction_pct=0.659212 moved_to_replicas=190978 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 32773, 32721, 31822, 32773, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31822 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64020 max_load_reduction_pct=0.659212 moved_to_replicas=190978 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[33096, 33096, 32773, 32721, 31822, 32773, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31822 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64020 max_load_reduction_pct=0.659212 moved_to_replicas=190978 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[35, 131024, 7, 15, 64795, 32784, 716, 32768] avg=32768.0000 max=131024 max_rank=1 min=7 min_rank=2 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[35, 131024, 7, 15, 64795, 32784, 716, 32768] avg=32768.0000 max=131024 max_rank=1 min=7 min_rank=2 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[35, 131024, 7, 15, 64795, 32784, 716, 32768] avg=32768.0000 max=131024 max_rank=1 min=7 min_rank=2 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[32443, 33096, 32767, 32775, 33096, 32784, 32415, 32768] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97928 max_load_reduction_pct=0.747405 moved_to_replicas=258612 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[35, 131024, 7, 15, 64795, 32784, 716, 32768] avg=32768.0000 max=131024 max_rank=1 min=7 min_rank=2 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[32443, 33096, 32767, 32775, 33096, 32784, 32415, 32768] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97928 max_load_reduction_pct=0.747405 moved_to_replicas=258612 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[35, 131024, 7, 15, 64795, 32784, 716, 32768] avg=32768.0000 max=131024 max_rank=1 min=7 min_rank=2 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[35, 131024, 7, 15, 64795, 32784, 716, 32768] avg=32768.0000 max=131024 max_rank=1 min=7 min_rank=2 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[35, 131024, 7, 15, 64795, 32784, 716, 32768] avg=32768.0000 max=131024 max_rank=1 min=7 min_rank=2 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[35, 131024, 7, 15, 64795, 32784, 716, 32768] avg=32768.0000 max=131024 max_rank=1 min=7 min_rank=2 imbalance_ratio=3.998535 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[32443, 33096, 32767, 32775, 33096, 32784, 32415, 32768] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97928 max_load_reduction_pct=0.747405 moved_to_replicas=258612 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[32443, 33096, 32767, 32775, 33096, 32784, 32415, 32768] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97928 max_load_reduction_pct=0.747405 moved_to_replicas=258612 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[32443, 33096, 32767, 32775, 33096, 32784, 32415, 32768] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97928 max_load_reduction_pct=0.747405 moved_to_replicas=258612 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[32443, 33096, 32767, 32775, 33096, 32784, 32415, 32768] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97928 max_load_reduction_pct=0.747405 moved_to_replicas=258612 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[13, 32776, 65539, 32764, 7, 32776, 32745, 65524] avg=32768.0000 max=65539 max_rank=2 min=7 min_rank=4 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[32443, 33096, 32767, 32775, 33096, 32784, 32415, 32768] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97928 max_load_reduction_pct=0.747405 moved_to_replicas=258612 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[32443, 33096, 32767, 32775, 33096, 32784, 32415, 32768] avg=32768.0000 max=33096 max_rank=1 min=32415 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97928 max_load_reduction_pct=0.747405 moved_to_replicas=258612 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[13, 32776, 65539, 32764, 7, 32776, 32745, 65524] avg=32768.0000 max=65539 max_rank=2 min=7 min_rank=4 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[13, 32776, 65539, 32764, 7, 32776, 32745, 65524] avg=32768.0000 max=65539 max_rank=2 min=7 min_rank=4 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[32441, 32776, 33096, 32764, 32450, 32776, 32745, 33096] avg=32768.0000 max=33096 max_rank=2 min=32441 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129737 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[13, 32776, 65539, 32764, 7, 32776, 32745, 65524] avg=32768.0000 max=65539 max_rank=2 min=7 min_rank=4 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[13, 32776, 65539, 32764, 7, 32776, 32745, 65524] avg=32768.0000 max=65539 max_rank=2 min=7 min_rank=4 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[32441, 32776, 33096, 32764, 32450, 32776, 32745, 33096] avg=32768.0000 max=33096 max_rank=2 min=32441 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129737 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[13, 32776, 65539, 32764, 7, 32776, 32745, 65524] avg=32768.0000 max=65539 max_rank=2 min=7 min_rank=4 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[13, 32776, 65539, 32764, 7, 32776, 32745, 65524] avg=32768.0000 max=65539 max_rank=2 min=7 min_rank=4 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[13, 32776, 65539, 32764, 7, 32776, 32745, 65524] avg=32768.0000 max=65539 max_rank=2 min=7 min_rank=4 imbalance_ratio=2.000092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[32441, 32776, 33096, 32764, 32450, 32776, 32745, 33096] avg=32768.0000 max=33096 max_rank=2 min=32441 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129737 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[32441, 32776, 33096, 32764, 32450, 32776, 32745, 33096] avg=32768.0000 max=33096 max_rank=2 min=32441 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129737 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[32441, 32776, 33096, 32764, 32450, 32776, 32745, 33096] avg=32768.0000 max=33096 max_rank=2 min=32441 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129737 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32755, 161, 20, 38, 65524, 98237, 32635] avg=32768.0000 max=98237 max_rank=6 min=20 min_rank=3 imbalance_ratio=2.997955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[32441, 32776, 33096, 32764, 32450, 32776, 32745, 33096] avg=32768.0000 max=33096 max_rank=2 min=32441 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129737 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[32441, 32776, 33096, 32764, 32450, 32776, 32745, 33096] avg=32768.0000 max=33096 max_rank=2 min=32441 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129737 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[32441, 32776, 33096, 32764, 32450, 32776, 32745, 33096] avg=32768.0000 max=33096 max_rank=2 min=32441 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32443 max_load_reduction_pct=0.495018 moved_to_replicas=129737 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32755, 161, 20, 38, 65524, 98237, 32635] avg=32768.0000 max=98237 max_rank=6 min=20 min_rank=3 imbalance_ratio=2.997955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32755, 161, 20, 38, 65524, 98237, 32635] avg=32768.0000 max=98237 max_rank=6 min=20 min_rank=3 imbalance_ratio=2.997955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32755, 32589, 32776, 32423, 33096, 33096, 32635] avg=32768.0000 max=33096 max_rank=5 min=32423 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65141 max_load_reduction_pct=0.663100 moved_to_replicas=195105 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32755, 161, 20, 38, 65524, 98237, 32635] avg=32768.0000 max=98237 max_rank=6 min=20 min_rank=3 imbalance_ratio=2.997955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32755, 161, 20, 38, 65524, 98237, 32635] avg=32768.0000 max=98237 max_rank=6 min=20 min_rank=3 imbalance_ratio=2.997955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32755, 32589, 32776, 32423, 33096, 33096, 32635] avg=32768.0000 max=33096 max_rank=5 min=32423 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65141 max_load_reduction_pct=0.663100 moved_to_replicas=195105 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32755, 161, 20, 38, 65524, 98237, 32635] avg=32768.0000 max=98237 max_rank=6 min=20 min_rank=3 imbalance_ratio=2.997955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32755, 161, 20, 38, 65524, 98237, 32635] avg=32768.0000 max=98237 max_rank=6 min=20 min_rank=3 imbalance_ratio=2.997955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32755, 32589, 32776, 32423, 33096, 33096, 32635] avg=32768.0000 max=33096 max_rank=5 min=32423 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65141 max_load_reduction_pct=0.663100 moved_to_replicas=195105 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32755, 161, 20, 38, 65524, 98237, 32635] avg=32768.0000 max=98237 max_rank=6 min=20 min_rank=3 imbalance_ratio=2.997955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32755, 32589, 32776, 32423, 33096, 33096, 32635] avg=32768.0000 max=33096 max_rank=5 min=32423 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65141 max_load_reduction_pct=0.663100 moved_to_replicas=195105 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[65160, 119, 32725, 17, 32758, 332, 65519, 65514] avg=32768.0000 max=65519 max_rank=6 min=17 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32755, 32589, 32776, 32423, 33096, 33096, 32635] avg=32768.0000 max=33096 max_rank=5 min=32423 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65141 max_load_reduction_pct=0.663100 moved_to_replicas=195105 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32755, 32589, 32776, 32423, 33096, 33096, 32635] avg=32768.0000 max=33096 max_rank=5 min=32423 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65141 max_load_reduction_pct=0.663100 moved_to_replicas=195105 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32755, 32589, 32776, 32423, 33096, 33096, 32635] avg=32768.0000 max=33096 max_rank=5 min=32423 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65141 max_load_reduction_pct=0.663100 moved_to_replicas=195105 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[32774, 32755, 32589, 32776, 32423, 33096, 33096, 32635] avg=32768.0000 max=33096 max_rank=5 min=32423 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65141 max_load_reduction_pct=0.663100 moved_to_replicas=195105 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[65160, 119, 32725, 17, 32758, 332, 65519, 65514] avg=32768.0000 max=65519 max_rank=6 min=17 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[65160, 119, 32725, 17, 32758, 332, 65519, 65514] avg=32768.0000 max=65519 max_rank=6 min=17 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[33096, 32537, 32725, 32440, 32758, 32396, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32396 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=193826 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[65160, 119, 32725, 17, 32758, 332, 65519, 65514] avg=32768.0000 max=65519 max_rank=6 min=17 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[65160, 119, 32725, 17, 32758, 332, 65519, 65514] avg=32768.0000 max=65519 max_rank=6 min=17 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[33096, 32537, 32725, 32440, 32758, 32396, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32396 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=193826 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[65160, 119, 32725, 17, 32758, 332, 65519, 65514] avg=32768.0000 max=65519 max_rank=6 min=17 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[65160, 119, 32725, 17, 32758, 332, 65519, 65514] avg=32768.0000 max=65519 max_rank=6 min=17 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[33096, 32537, 32725, 32440, 32758, 32396, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32396 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=193826 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[65160, 119, 32725, 17, 32758, 332, 65519, 65514] avg=32768.0000 max=65519 max_rank=6 min=17 min_rank=3 imbalance_ratio=1.999481 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[33096, 32537, 32725, 32440, 32758, 32396, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32396 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=193826 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[33096, 32537, 32725, 32440, 32758, 32396, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32396 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=193826 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[33096, 32537, 32725, 32440, 32758, 32396, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32396 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=193826 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[33096, 32537, 32725, 32440, 32758, 32396, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32396 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=193826 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[33096, 32537, 32725, 32440, 32758, 32396, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32396 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32423 max_load_reduction_pct=0.494864 moved_to_replicas=193826 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=32 total_expected_recv=1049513 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=250880 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=32 total_expected_recv=1047592 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=252672 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=32 total_expected_recv=1048709 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=251648 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=32 total_expected_recv=1047443 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253952 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=32 total_expected_recv=1044751 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254592 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=32 total_expected_recv=1050505 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=249984 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=32 total_expected_recv=1050624 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=248064 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=32 total_expected_recv=1049471 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=251264 peak_demand=33097 required_capacity=1.010040 + [2026-08-30 05:10:29] iteration 7/ 12 | consumed samples: 112 | elapsed time per iteration (ms): 560.0 | throughput per GPU (TFLOP/s/GPU): 595.3 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 1.012984E+01 | loss scale: 1.0 | grad norm: 7.725 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=40960 expected_recv=65363 expected_overflow=0 exact_recv_per_rank=[31862, 32267, 32536, 33096, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=40960 expected_recv=64958 expected_overflow=0 exact_recv_per_rank=[31862, 32267, 32536, 33096, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31862, 32267, 32536, 33096, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=40960 expected_recv=64660 expected_overflow=0 exact_recv_per_rank=[31862, 32267, 32536, 33096, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31862, 32267, 32536, 33096, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[31862, 32267, 32536, 33096, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31862, 32267, 32536, 33096, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=2 budget=40960 expected_recv=65363 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=2 budget=40960 expected_recv=64958 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=2 budget=40960 expected_recv=64660 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=40960 expected_recv=64540 expected_overflow=0 exact_recv_per_rank=[31862, 32267, 32536, 33096, 33096, 33096, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=2 budget=40960 expected_recv=64540 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32803, 32835, 32759, 32424, 32552, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32803, 32835, 32759, 32424, 32552, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65587 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32803, 32835, 32759, 32424, 32552, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65654 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32803, 32835, 32759, 32424, 32552, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=40960 expected_recv=64860 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32803, 32835, 32759, 32424, 32552, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65527 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=40960 expected_recv=66008 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32803, 32835, 32759, 32424, 32552, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65587 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65654 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=40960 expected_recv=64929 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32803, 32835, 32759, 32424, 32552, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=2 budget=40960 expected_recv=64860 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[33096, 32763, 32803, 32835, 32759, 32424, 32552, 32912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=2 budget=40960 expected_recv=66008 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=2 budget=40960 expected_recv=64929 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=2 budget=40960 expected_recv=65531 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[33096, 32770, 33096, 32771, 32774, 32372, 32738, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32770, 33096, 32771, 32774, 32372, 32738, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[33096, 32770, 33096, 32771, 32774, 32372, 32738, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=40960 expected_recv=64758 expected_overflow=0 exact_recv_per_rank=[33096, 32770, 33096, 32771, 32774, 32372, 32738, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32770, 33096, 32771, 32774, 32372, 32738, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65535 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65042 expected_overflow=0 exact_recv_per_rank=[33096, 32770, 33096, 32771, 32774, 32372, 32738, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65540 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65489 expected_overflow=0 exact_recv_per_rank=[33096, 32770, 33096, 32771, 32774, 32372, 32738, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[33096, 32770, 33096, 32771, 32774, 32372, 32738, 32527] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=2 budget=40960 expected_recv=64758 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65042 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65489 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=2 budget=40960 expected_recv=65540 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=40960 expected_recv=62953 expected_overflow=0 exact_recv_per_rank=[32654, 31486, 33095, 32769, 33096, 32852, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65368 expected_overflow=0 exact_recv_per_rank=[32654, 31486, 33095, 32769, 33096, 32852, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[32654, 31486, 33095, 32769, 33096, 32852, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65665 expected_overflow=0 exact_recv_per_rank=[32654, 31486, 33095, 32769, 33096, 32852, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32654, 31486, 33095, 32769, 33096, 32852, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65368 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=2 budget=40960 expected_recv=62953 actual_valid_recv=63488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32654, 31486, 33095, 32769, 33096, 32852, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32654, 31486, 33095, 32769, 33096, 32852, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65536 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=2 budget=40960 expected_recv=65665 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32654, 31486, 33095, 32769, 33096, 32852, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=63488 slack_slots=18432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=40960 expected_recv=64235 expected_overflow=0 exact_recv_per_rank=[32087, 32772, 33096, 32778, 32741, 32769, 32805, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[32087, 32772, 33096, 32778, 32741, 32769, 32805, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[32087, 32772, 33096, 32778, 32741, 32769, 32805, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=2 budget=40960 expected_recv=64235 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[32087, 32772, 33096, 32778, 32741, 32769, 32805, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65542 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65546 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32087, 32772, 33096, 32778, 32741, 32769, 32805, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=40960 expected_recv=66134 expected_overflow=0 exact_recv_per_rank=[32087, 32772, 33096, 32778, 32741, 32769, 32805, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65598 expected_overflow=0 exact_recv_per_rank=[32087, 32772, 33096, 32778, 32741, 32769, 32805, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65537 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65504 expected_overflow=0 exact_recv_per_rank=[32087, 32772, 33096, 32778, 32741, 32769, 32805, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=2 budget=40960 expected_recv=66134 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65598 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=2 budget=40960 expected_recv=65504 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65451 expected_overflow=0 exact_recv_per_rank=[32721, 32855, 32763, 32769, 32402, 33096, 32770, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65451 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65663 expected_overflow=0 exact_recv_per_rank=[32721, 32855, 32763, 32769, 32402, 33096, 32770, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[32721, 32855, 32763, 32769, 32402, 33096, 32770, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32721, 32855, 32763, 32769, 32402, 33096, 32770, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[32721, 32855, 32763, 32769, 32402, 33096, 32770, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65663 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65540 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65549 expected_overflow=0 exact_recv_per_rank=[32721, 32855, 32763, 32769, 32402, 33096, 32770, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65521 expected_overflow=0 exact_recv_per_rank=[32721, 32855, 32763, 32769, 32402, 33096, 32770, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=40960 expected_recv=64837 expected_overflow=0 exact_recv_per_rank=[32721, 32855, 32763, 32769, 32402, 33096, 32770, 32768] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65535 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65549 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=2 budget=40960 expected_recv=65521 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=2 budget=40960 expected_recv=64837 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32290, 33096, 33096, 32776, 32262, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32290, 33096, 33096, 32776, 32262, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=40960 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32290, 33096, 33096, 32776, 32262, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32290, 33096, 33096, 32776, 32262, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=40960 expected_recv=64796 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32290, 33096, 33096, 32776, 32262, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=40960 expected_recv=64698 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32290, 33096, 33096, 32776, 32262, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=40960 expected_recv=64486 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32290, 33096, 33096, 32776, 32262, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32290, 33096, 33096, 32776, 32262, 32432] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=2 budget=40960 expected_recv=65540 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=2 budget=40960 expected_recv=64796 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=2 budget=40960 expected_recv=64698 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=2 budget=40960 expected_recv=64486 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=40960 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32793, 32438, 33096, 33096, 32412, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=2 budget=40960 expected_recv=65532 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32793, 32438, 33096, 33096, 32412, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32793, 32438, 33096, 33096, 32412, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=40960 expected_recv=64838 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32793, 32438, 33096, 33096, 32412, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=40960 expected_recv=64852 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32793, 32438, 33096, 33096, 32412, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=40960 expected_recv=64866 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32793, 32438, 33096, 33096, 32412, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=40960 expected_recv=65624 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32793, 32438, 33096, 33096, 32412, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=2 budget=40960 expected_recv=64838 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32776, 33096, 32793, 32438, 33096, 33096, 32412, 32437] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=2 budget=40960 expected_recv=64852 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=2 budget=40960 expected_recv=64866 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=2 budget=40960 expected_recv=65624 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33097, 32761, 33096, 32776, 32450, 32777, 32818, 32369] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[33097, 32761, 33096, 32776, 32450, 32777, 32818, 32369] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[33097, 32761, 33096, 32776, 32450, 32777, 32818, 32369] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65529 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65543 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[33097, 32761, 33096, 32776, 32450, 32777, 32818, 32369] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65585 expected_overflow=0 exact_recv_per_rank=[33097, 32761, 33096, 32776, 32450, 32777, 32818, 32369] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=40960 expected_recv=64721 expected_overflow=0 exact_recv_per_rank=[33097, 32761, 33096, 32776, 32450, 32777, 32818, 32369] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33097, 32761, 33096, 32776, 32450, 32777, 32818, 32369] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65545 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=2 budget=40960 expected_recv=65585 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=40960 expected_recv=64979 expected_overflow=0 exact_recv_per_rank=[33097, 32761, 33096, 32776, 32450, 32777, 32818, 32369] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=2 budget=40960 expected_recv=64721 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[32774, 32774, 32755, 32722, 32451, 33094, 33096, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=2 budget=40960 expected_recv=64979 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65542 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[32774, 32774, 32755, 32722, 32451, 33094, 33096, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65543 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65449 expected_overflow=0 exact_recv_per_rank=[32774, 32774, 32755, 32722, 32451, 33094, 33096, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65449 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32774, 32774, 32755, 32722, 32451, 33094, 33096, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32774, 32774, 32755, 32722, 32451, 33094, 33096, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65513 expected_overflow=0 exact_recv_per_rank=[32774, 32774, 32755, 32722, 32451, 33094, 33096, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=40960 expected_recv=64954 expected_overflow=0 exact_recv_per_rank=[32774, 32774, 32755, 32722, 32451, 33094, 33096, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=40960 expected_recv=64902 expected_overflow=0 exact_recv_per_rank=[32774, 32774, 32755, 32722, 32451, 33094, 33096, 32478] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=2 budget=40960 expected_recv=65513 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32438, 32431, 32773, 33096, 32767, 33096, 32447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=2 budget=40960 expected_recv=64954 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=2 budget=40960 expected_recv=64902 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=40960 expected_recv=64878 expected_overflow=0 exact_recv_per_rank=[33096, 32438, 32431, 32773, 33096, 32767, 33096, 32447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=2 budget=40960 expected_recv=64878 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=40960 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[33096, 32438, 32431, 32773, 33096, 32767, 33096, 32447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=40960 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[33096, 32438, 32431, 32773, 33096, 32767, 33096, 32447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=2 budget=40960 expected_recv=65541 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32438, 32431, 32773, 33096, 32767, 33096, 32447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=40960 expected_recv=64871 expected_overflow=0 exact_recv_per_rank=[33096, 32438, 32431, 32773, 33096, 32767, 33096, 32447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=40960 expected_recv=64887 expected_overflow=0 exact_recv_per_rank=[33096, 32438, 32431, 32773, 33096, 32767, 33096, 32447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32438, 32431, 32773, 33096, 32767, 33096, 32447] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=2 budget=40960 expected_recv=65535 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=2 budget=40960 expected_recv=64871 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33096, 33025, 33096, 33096, 31857, 32768, 32446, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=2 budget=40960 expected_recv=64887 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=40960 expected_recv=65951 expected_overflow=0 exact_recv_per_rank=[33096, 33025, 33096, 33096, 31857, 32768, 32446, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=2 budget=40960 expected_recv=65951 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33025, 33096, 33096, 31857, 32768, 32446, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=40960 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[33096, 33025, 33096, 33096, 31857, 32768, 32446, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=40960 expected_recv=64398 expected_overflow=0 exact_recv_per_rank=[33096, 33025, 33096, 33096, 31857, 32768, 32446, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33025, 33096, 33096, 31857, 32768, 32446, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=2 budget=40960 expected_recv=65536 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=40960 expected_recv=65528 expected_overflow=0 exact_recv_per_rank=[33096, 33025, 33096, 33096, 31857, 32768, 32446, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=2 budget=40960 expected_recv=64398 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=40960 expected_recv=64297 expected_overflow=0 exact_recv_per_rank=[33096, 33025, 33096, 33096, 31857, 32768, 32446, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=2 budget=40960 expected_recv=65528 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=40960 expected_recv=64923 expected_overflow=0 exact_recv_per_rank=[32423, 33096, 32778, 32397, 33096, 33096, 32971, 32287] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=2 budget=40960 expected_recv=64297 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=12 calls=2 budget=40960 expected_recv=64923 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32423, 33096, 32778, 32397, 33096, 33096, 32971, 32287] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65164 expected_overflow=0 exact_recv_per_rank=[32423, 33096, 32778, 32397, 33096, 33096, 32971, 32287] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32423, 33096, 32778, 32397, 33096, 33096, 32971, 32287] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65164 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65739 expected_overflow=0 exact_recv_per_rank=[32423, 33096, 32778, 32397, 33096, 33096, 32971, 32287] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65739 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=40960 expected_recv=64668 expected_overflow=0 exact_recv_per_rank=[32423, 33096, 32778, 32397, 33096, 33096, 32971, 32287] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65864 expected_overflow=0 exact_recv_per_rank=[32423, 33096, 32778, 32397, 33096, 33096, 32971, 32287] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[32423, 33096, 32778, 32397, 33096, 33096, 32971, 32287] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=40960 expected_recv=64911 expected_overflow=0 exact_recv_per_rank=[32470, 32432, 33096, 32414, 33096, 32781, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=2 budget=40960 expected_recv=64668 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65864 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=12 calls=2 budget=40960 expected_recv=65546 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=40960 expected_recv=64872 expected_overflow=0 exact_recv_per_rank=[32470, 32432, 33096, 32414, 33096, 32781, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=2 budget=40960 expected_recv=64911 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=2 budget=40960 expected_recv=64872 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=40960 expected_recv=65549 expected_overflow=0 exact_recv_per_rank=[32470, 32432, 33096, 32414, 33096, 32781, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=40960 expected_recv=64853 expected_overflow=0 exact_recv_per_rank=[32470, 32432, 33096, 32414, 33096, 32781, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=2 budget=40960 expected_recv=65549 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=13 calls=2 budget=40960 expected_recv=64853 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=40960 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[32470, 32432, 33096, 32414, 33096, 32781, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32470, 32432, 33096, 32414, 33096, 32781, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32470, 32432, 33096, 32414, 33096, 32781, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32470, 32432, 33096, 32414, 33096, 32781, 32759, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=2 budget=40960 expected_recv=65527 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65549 expected_overflow=0 exact_recv_per_rank=[32780, 32756, 32782, 32778, 32472, 33096, 33096, 32384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65524 expected_overflow=0 exact_recv_per_rank=[32780, 32756, 32782, 32778, 32472, 33096, 33096, 32384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65549 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65524 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32780, 32756, 32782, 32778, 32472, 33096, 33096, 32384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[32780, 32756, 32782, 32778, 32472, 33096, 33096, 32384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65545 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32780, 32756, 32782, 32778, 32472, 33096, 33096, 32384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=40960 expected_recv=64897 expected_overflow=0 exact_recv_per_rank=[32780, 32756, 32782, 32778, 32472, 33096, 33096, 32384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=40960 expected_recv=64842 expected_overflow=0 exact_recv_per_rank=[32780, 32756, 32782, 32778, 32472, 33096, 33096, 32384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65547 expected_overflow=0 exact_recv_per_rank=[32780, 32756, 32782, 32778, 32472, 33096, 33096, 32384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=2 budget=40960 expected_recv=64897 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=2 budget=40960 expected_recv=64842 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32836, 33042, 32758, 32767, 31782, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=14 calls=2 budget=40960 expected_recv=65547 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=40960 expected_recv=65598 expected_overflow=0 exact_recv_per_rank=[33096, 32836, 33042, 32758, 32767, 31782, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=40960 expected_recv=63878 expected_overflow=0 exact_recv_per_rank=[33096, 32836, 33042, 32758, 32767, 31782, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=15 calls=2 budget=40960 expected_recv=65598 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=40960 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[33096, 32836, 33042, 32758, 32767, 31782, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=2 budget=40960 expected_recv=63878 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=15 calls=2 budget=40960 expected_recv=65526 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=40960 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[33096, 32836, 33042, 32758, 32767, 31782, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=2 budget=40960 expected_recv=65535 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=40960 expected_recv=65229 expected_overflow=0 exact_recv_per_rank=[33096, 32836, 33042, 32758, 32767, 31782, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32836, 33042, 32758, 32767, 31782, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=40960 expected_recv=66138 expected_overflow=0 exact_recv_per_rank=[33096, 32836, 33042, 32758, 32767, 31782, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=15 calls=2 budget=40960 expected_recv=65229 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=15 calls=2 budget=40960 expected_recv=66138 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[21963, 30577, 15089, 38952, 28930, 8365, 75346, 42922] avg=32768.0000 max=75346 max_rank=6 min=8365 min_rank=5 imbalance_ratio=2.299377 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[21963, 30577, 15089, 38952, 28930, 8365, 75346, 42922] avg=32768.0000 max=75346 max_rank=6 min=8365 min_rank=5 imbalance_ratio=2.299377 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[21963, 30577, 15089, 38952, 28930, 8365, 75346, 42922] avg=32768.0000 max=75346 max_rank=6 min=8365 min_rank=5 imbalance_ratio=2.299377 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[21963, 30577, 15089, 38952, 28930, 8365, 75346, 42922] avg=32768.0000 max=75346 max_rank=6 min=8365 min_rank=5 imbalance_ratio=2.299377 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[31862, 32267, 32536, 33096, 33096, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=3 min=31862 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42250 max_load_reduction_pct=0.560746 moved_to_replicas=114473 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[31862, 32267, 32536, 33096, 33096, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=3 min=31862 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42250 max_load_reduction_pct=0.560746 moved_to_replicas=114473 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[21963, 30577, 15089, 38952, 28930, 8365, 75346, 42922] avg=32768.0000 max=75346 max_rank=6 min=8365 min_rank=5 imbalance_ratio=2.299377 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[31862, 32267, 32536, 33096, 33096, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=3 min=31862 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42250 max_load_reduction_pct=0.560746 moved_to_replicas=114473 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[21963, 30577, 15089, 38952, 28930, 8365, 75346, 42922] avg=32768.0000 max=75346 max_rank=6 min=8365 min_rank=5 imbalance_ratio=2.299377 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[31862, 32267, 32536, 33096, 33096, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=3 min=31862 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42250 max_load_reduction_pct=0.560746 moved_to_replicas=114473 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[21963, 30577, 15089, 38952, 28930, 8365, 75346, 42922] avg=32768.0000 max=75346 max_rank=6 min=8365 min_rank=5 imbalance_ratio=2.299377 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[21963, 30577, 15089, 38952, 28930, 8365, 75346, 42922] avg=32768.0000 max=75346 max_rank=6 min=8365 min_rank=5 imbalance_ratio=2.299377 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[31862, 32267, 32536, 33096, 33096, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=3 min=31862 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42250 max_load_reduction_pct=0.560746 moved_to_replicas=114473 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[65501, 32763, 32803, 32835, 32759, 19, 32552, 32912] avg=32768.0000 max=65501 max_rank=0 min=19 min_rank=5 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[65501, 32763, 32803, 32835, 32759, 19, 32552, 32912] avg=32768.0000 max=65501 max_rank=0 min=19 min_rank=5 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[65501, 32763, 32803, 32835, 32759, 19, 32552, 32912] avg=32768.0000 max=65501 max_rank=0 min=19 min_rank=5 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[31862, 32267, 32536, 33096, 33096, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=3 min=31862 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42250 max_load_reduction_pct=0.560746 moved_to_replicas=114473 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[31862, 32267, 32536, 33096, 33096, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=3 min=31862 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42250 max_load_reduction_pct=0.560746 moved_to_replicas=114473 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[31862, 32267, 32536, 33096, 33096, 33096, 33096, 33095] avg=32768.0000 max=33096 max_rank=3 min=31862 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=42250 max_load_reduction_pct=0.560746 moved_to_replicas=114473 active_clone_links=13 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[65501, 32763, 32803, 32835, 32759, 19, 32552, 32912] avg=32768.0000 max=65501 max_rank=0 min=19 min_rank=5 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[33096, 32763, 32803, 32835, 32759, 32424, 32552, 32912] avg=32768.0000 max=33096 max_rank=0 min=32424 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.494725 moved_to_replicas=64863 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[33096, 32763, 32803, 32835, 32759, 32424, 32552, 32912] avg=32768.0000 max=33096 max_rank=0 min=32424 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.494725 moved_to_replicas=64863 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[33096, 32763, 32803, 32835, 32759, 32424, 32552, 32912] avg=32768.0000 max=33096 max_rank=0 min=32424 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.494725 moved_to_replicas=64863 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[65501, 32763, 32803, 32835, 32759, 19, 32552, 32912] avg=32768.0000 max=65501 max_rank=0 min=19 min_rank=5 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[65501, 32763, 32803, 32835, 32759, 19, 32552, 32912] avg=32768.0000 max=65501 max_rank=0 min=19 min_rank=5 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[33096, 32763, 32803, 32835, 32759, 32424, 32552, 32912] avg=32768.0000 max=33096 max_rank=0 min=32424 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.494725 moved_to_replicas=64863 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[65501, 32763, 32803, 32835, 32759, 19, 32552, 32912] avg=32768.0000 max=65501 max_rank=0 min=19 min_rank=5 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[65501, 32763, 32803, 32835, 32759, 19, 32552, 32912] avg=32768.0000 max=65501 max_rank=0 min=19 min_rank=5 imbalance_ratio=1.998932 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[33096, 32763, 32803, 32835, 32759, 32424, 32552, 32912] avg=32768.0000 max=33096 max_rank=0 min=32424 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.494725 moved_to_replicas=64863 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[65471, 32770, 98193, 32771, 15, 34, 32738, 152] avg=32768.0000 max=98193 max_rank=2 min=15 min_rank=4 imbalance_ratio=2.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[33096, 32763, 32803, 32835, 32759, 32424, 32552, 32912] avg=32768.0000 max=33096 max_rank=0 min=32424 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.494725 moved_to_replicas=64863 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[65471, 32770, 98193, 32771, 15, 34, 32738, 152] avg=32768.0000 max=98193 max_rank=2 min=15 min_rank=4 imbalance_ratio=2.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[33096, 32763, 32803, 32835, 32759, 32424, 32552, 32912] avg=32768.0000 max=33096 max_rank=0 min=32424 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.494725 moved_to_replicas=64863 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[65471, 32770, 98193, 32771, 15, 34, 32738, 152] avg=32768.0000 max=98193 max_rank=2 min=15 min_rank=4 imbalance_ratio=2.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[33096, 32763, 32803, 32835, 32759, 32424, 32552, 32912] avg=32768.0000 max=33096 max_rank=0 min=32424 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32405 max_load_reduction_pct=0.494725 moved_to_replicas=64863 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[65471, 32770, 98193, 32771, 15, 34, 32738, 152] avg=32768.0000 max=98193 max_rank=2 min=15 min_rank=4 imbalance_ratio=2.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32770, 33096, 32771, 32774, 32372, 32738, 32527] avg=32768.0000 max=33096 max_rank=0 min=32372 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=195017 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32770, 33096, 32771, 32774, 32372, 32738, 32527] avg=32768.0000 max=33096 max_rank=0 min=32372 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=195017 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[65471, 32770, 98193, 32771, 15, 34, 32738, 152] avg=32768.0000 max=98193 max_rank=2 min=15 min_rank=4 imbalance_ratio=2.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32770, 33096, 32771, 32774, 32372, 32738, 32527] avg=32768.0000 max=33096 max_rank=0 min=32372 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=195017 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[65471, 32770, 98193, 32771, 15, 34, 32738, 152] avg=32768.0000 max=98193 max_rank=2 min=15 min_rank=4 imbalance_ratio=2.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32770, 33096, 32771, 32774, 32372, 32738, 32527] avg=32768.0000 max=33096 max_rank=0 min=32372 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=195017 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[65471, 32770, 98193, 32771, 15, 34, 32738, 152] avg=32768.0000 max=98193 max_rank=2 min=15 min_rank=4 imbalance_ratio=2.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[65471, 32770, 98193, 32771, 15, 34, 32738, 152] avg=32768.0000 max=98193 max_rank=2 min=15 min_rank=4 imbalance_ratio=2.996613 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32770, 33096, 32771, 32774, 32372, 32738, 32527] avg=32768.0000 max=33096 max_rank=0 min=32372 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=195017 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[32654, 11520, 44619, 8, 65519, 32852, 69713, 5259] avg=32768.0000 max=69713 max_rank=6 min=8 min_rank=3 imbalance_ratio=2.127472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[32654, 11520, 44619, 8, 65519, 32852, 69713, 5259] avg=32768.0000 max=69713 max_rank=6 min=8 min_rank=3 imbalance_ratio=2.127472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32770, 33096, 32771, 32774, 32372, 32738, 32527] avg=32768.0000 max=33096 max_rank=0 min=32372 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=195017 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32770, 33096, 32771, 32774, 32372, 32738, 32527] avg=32768.0000 max=33096 max_rank=0 min=32372 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=195017 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[32654, 11520, 44619, 8, 65519, 32852, 69713, 5259] avg=32768.0000 max=69713 max_rank=6 min=8 min_rank=3 imbalance_ratio=2.127472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32770, 33096, 32771, 32774, 32372, 32738, 32527] avg=32768.0000 max=33096 max_rank=0 min=32372 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=65097 max_load_reduction_pct=0.662949 moved_to_replicas=195017 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[32654, 11520, 44619, 8, 65519, 32852, 69713, 5259] avg=32768.0000 max=69713 max_rank=6 min=8 min_rank=3 imbalance_ratio=2.127472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[32654, 31486, 33095, 32769, 33096, 32852, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31486 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36617 max_load_reduction_pct=0.525254 moved_to_replicas=159420 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[32654, 31486, 33095, 32769, 33096, 32852, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31486 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36617 max_load_reduction_pct=0.525254 moved_to_replicas=159420 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[32654, 11520, 44619, 8, 65519, 32852, 69713, 5259] avg=32768.0000 max=69713 max_rank=6 min=8 min_rank=3 imbalance_ratio=2.127472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[32654, 31486, 33095, 32769, 33096, 32852, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31486 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36617 max_load_reduction_pct=0.525254 moved_to_replicas=159420 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[32654, 11520, 44619, 8, 65519, 32852, 69713, 5259] avg=32768.0000 max=69713 max_rank=6 min=8 min_rank=3 imbalance_ratio=2.127472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[32654, 31486, 33095, 32769, 33096, 32852, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31486 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36617 max_load_reduction_pct=0.525254 moved_to_replicas=159420 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[32654, 11520, 44619, 8, 65519, 32852, 69713, 5259] avg=32768.0000 max=69713 max_rank=6 min=8 min_rank=3 imbalance_ratio=2.127472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[32654, 11520, 44619, 8, 65519, 32852, 69713, 5259] avg=32768.0000 max=69713 max_rank=6 min=8 min_rank=3 imbalance_ratio=2.127472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[32654, 31486, 33095, 32769, 33096, 32852, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31486 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36617 max_load_reduction_pct=0.525254 moved_to_replicas=159420 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[20, 32772, 33259, 32778, 32741, 32769, 32805, 65000] avg=32768.0000 max=65000 max_rank=7 min=20 min_rank=0 imbalance_ratio=1.983643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[20, 32772, 33259, 32778, 32741, 32769, 32805, 65000] avg=32768.0000 max=65000 max_rank=7 min=20 min_rank=0 imbalance_ratio=1.983643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[32654, 31486, 33095, 32769, 33096, 32852, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31486 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36617 max_load_reduction_pct=0.525254 moved_to_replicas=159420 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[20, 32772, 33259, 32778, 32741, 32769, 32805, 65000] avg=32768.0000 max=65000 max_rank=7 min=20 min_rank=0 imbalance_ratio=1.983643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[32654, 31486, 33095, 32769, 33096, 32852, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31486 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36617 max_load_reduction_pct=0.525254 moved_to_replicas=159420 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[32654, 31486, 33095, 32769, 33096, 32852, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31486 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=36617 max_load_reduction_pct=0.525254 moved_to_replicas=159420 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[32087, 32772, 33096, 32778, 32741, 32769, 32805, 33096] avg=32768.0000 max=33096 max_rank=2 min=32087 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31904 max_load_reduction_pct=0.490831 moved_to_replicas=64214 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[20, 32772, 33259, 32778, 32741, 32769, 32805, 65000] avg=32768.0000 max=65000 max_rank=7 min=20 min_rank=0 imbalance_ratio=1.983643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[32087, 32772, 33096, 32778, 32741, 32769, 32805, 33096] avg=32768.0000 max=33096 max_rank=2 min=32087 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31904 max_load_reduction_pct=0.490831 moved_to_replicas=64214 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[32087, 32772, 33096, 32778, 32741, 32769, 32805, 33096] avg=32768.0000 max=33096 max_rank=2 min=32087 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31904 max_load_reduction_pct=0.490831 moved_to_replicas=64214 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[20, 32772, 33259, 32778, 32741, 32769, 32805, 65000] avg=32768.0000 max=65000 max_rank=7 min=20 min_rank=0 imbalance_ratio=1.983643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[20, 32772, 33259, 32778, 32741, 32769, 32805, 65000] avg=32768.0000 max=65000 max_rank=7 min=20 min_rank=0 imbalance_ratio=1.983643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[32087, 32772, 33096, 32778, 32741, 32769, 32805, 33096] avg=32768.0000 max=33096 max_rank=2 min=32087 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31904 max_load_reduction_pct=0.490831 moved_to_replicas=64214 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[20, 32772, 33259, 32778, 32741, 32769, 32805, 65000] avg=32768.0000 max=65000 max_rank=7 min=20 min_rank=0 imbalance_ratio=1.983643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[20, 32772, 33259, 32778, 32741, 32769, 32805, 65000] avg=32768.0000 max=65000 max_rank=7 min=20 min_rank=0 imbalance_ratio=1.983643 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[32721, 32855, 32763, 32769, 17, 98241, 32770, 8] avg=32768.0000 max=98241 max_rank=5 min=8 min_rank=7 imbalance_ratio=2.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[32087, 32772, 33096, 32778, 32741, 32769, 32805, 33096] avg=32768.0000 max=33096 max_rank=2 min=32087 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31904 max_load_reduction_pct=0.490831 moved_to_replicas=64214 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[32721, 32855, 32763, 32769, 17, 98241, 32770, 8] avg=32768.0000 max=98241 max_rank=5 min=8 min_rank=7 imbalance_ratio=2.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[32087, 32772, 33096, 32778, 32741, 32769, 32805, 33096] avg=32768.0000 max=33096 max_rank=2 min=32087 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31904 max_load_reduction_pct=0.490831 moved_to_replicas=64214 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[32721, 32855, 32763, 32769, 17, 98241, 32770, 8] avg=32768.0000 max=98241 max_rank=5 min=8 min_rank=7 imbalance_ratio=2.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[32087, 32772, 33096, 32778, 32741, 32769, 32805, 33096] avg=32768.0000 max=33096 max_rank=2 min=32087 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31904 max_load_reduction_pct=0.490831 moved_to_replicas=64214 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[32721, 32855, 32763, 32769, 32402, 33096, 32770, 32768] avg=32768.0000 max=33096 max_rank=5 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65145 max_load_reduction_pct=0.663114 moved_to_replicas=130344 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[32087, 32772, 33096, 32778, 32741, 32769, 32805, 33096] avg=32768.0000 max=33096 max_rank=2 min=32087 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31904 max_load_reduction_pct=0.490831 moved_to_replicas=64214 active_clone_links=3 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[32721, 32855, 32763, 32769, 17, 98241, 32770, 8] avg=32768.0000 max=98241 max_rank=5 min=8 min_rank=7 imbalance_ratio=2.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[32721, 32855, 32763, 32769, 32402, 33096, 32770, 32768] avg=32768.0000 max=33096 max_rank=5 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65145 max_load_reduction_pct=0.663114 moved_to_replicas=130344 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[32721, 32855, 32763, 32769, 32402, 33096, 32770, 32768] avg=32768.0000 max=33096 max_rank=5 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65145 max_load_reduction_pct=0.663114 moved_to_replicas=130344 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[32721, 32855, 32763, 32769, 17, 98241, 32770, 8] avg=32768.0000 max=98241 max_rank=5 min=8 min_rank=7 imbalance_ratio=2.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[32721, 32855, 32763, 32769, 17, 98241, 32770, 8] avg=32768.0000 max=98241 max_rank=5 min=8 min_rank=7 imbalance_ratio=2.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[32721, 32855, 32763, 32769, 32402, 33096, 32770, 32768] avg=32768.0000 max=33096 max_rank=5 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65145 max_load_reduction_pct=0.663114 moved_to_replicas=130344 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[32721, 32855, 32763, 32769, 17, 98241, 32770, 8] avg=32768.0000 max=98241 max_rank=5 min=8 min_rank=7 imbalance_ratio=2.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[65518, 23760, 32290, 43442, 64334, 32776, 14, 10] avg=32768.0000 max=65518 max_rank=0 min=10 min_rank=7 imbalance_ratio=1.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[32721, 32855, 32763, 32769, 32402, 33096, 32770, 32768] avg=32768.0000 max=33096 max_rank=5 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65145 max_load_reduction_pct=0.663114 moved_to_replicas=130344 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[32721, 32855, 32763, 32769, 17, 98241, 32770, 8] avg=32768.0000 max=98241 max_rank=5 min=8 min_rank=7 imbalance_ratio=2.998077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[65518, 23760, 32290, 43442, 64334, 32776, 14, 10] avg=32768.0000 max=65518 max_rank=0 min=10 min_rank=7 imbalance_ratio=1.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[65518, 23760, 32290, 43442, 64334, 32776, 14, 10] avg=32768.0000 max=65518 max_rank=0 min=10 min_rank=7 imbalance_ratio=1.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[32721, 32855, 32763, 32769, 32402, 33096, 32770, 32768] avg=32768.0000 max=33096 max_rank=5 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65145 max_load_reduction_pct=0.663114 moved_to_replicas=130344 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[32721, 32855, 32763, 32769, 32402, 33096, 32770, 32768] avg=32768.0000 max=33096 max_rank=5 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65145 max_load_reduction_pct=0.663114 moved_to_replicas=130344 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[33096, 33096, 32290, 33096, 33096, 32776, 32262, 32432] avg=32768.0000 max=33096 max_rank=0 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32422 max_load_reduction_pct=0.494856 moved_to_replicas=144593 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[65518, 23760, 32290, 43442, 64334, 32776, 14, 10] avg=32768.0000 max=65518 max_rank=0 min=10 min_rank=7 imbalance_ratio=1.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[32721, 32855, 32763, 32769, 32402, 33096, 32770, 32768] avg=32768.0000 max=33096 max_rank=5 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65145 max_load_reduction_pct=0.663114 moved_to_replicas=130344 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[33096, 33096, 32290, 33096, 33096, 32776, 32262, 32432] avg=32768.0000 max=33096 max_rank=0 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32422 max_load_reduction_pct=0.494856 moved_to_replicas=144593 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[33096, 33096, 32290, 33096, 33096, 32776, 32262, 32432] avg=32768.0000 max=33096 max_rank=0 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32422 max_load_reduction_pct=0.494856 moved_to_replicas=144593 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[65518, 23760, 32290, 43442, 64334, 32776, 14, 10] avg=32768.0000 max=65518 max_rank=0 min=10 min_rank=7 imbalance_ratio=1.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[65518, 23760, 32290, 43442, 64334, 32776, 14, 10] avg=32768.0000 max=65518 max_rank=0 min=10 min_rank=7 imbalance_ratio=1.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[33096, 33096, 32290, 33096, 33096, 32776, 32262, 32432] avg=32768.0000 max=33096 max_rank=0 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32422 max_load_reduction_pct=0.494856 moved_to_replicas=144593 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[65518, 23760, 32290, 43442, 64334, 32776, 14, 10] avg=32768.0000 max=65518 max_rank=0 min=10 min_rank=7 imbalance_ratio=1.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[32776, 65494, 32793, 571, 65522, 64963, 14, 11] avg=32768.0000 max=65522 max_rank=4 min=11 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[33096, 33096, 32290, 33096, 33096, 32776, 32262, 32432] avg=32768.0000 max=33096 max_rank=0 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32422 max_load_reduction_pct=0.494856 moved_to_replicas=144593 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[65518, 23760, 32290, 43442, 64334, 32776, 14, 10] avg=32768.0000 max=65518 max_rank=0 min=10 min_rank=7 imbalance_ratio=1.999451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[32776, 65494, 32793, 571, 65522, 64963, 14, 11] avg=32768.0000 max=65522 max_rank=4 min=11 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[32776, 65494, 32793, 571, 65522, 64963, 14, 11] avg=32768.0000 max=65522 max_rank=4 min=11 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[33096, 33096, 32290, 33096, 33096, 32776, 32262, 32432] avg=32768.0000 max=33096 max_rank=0 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32422 max_load_reduction_pct=0.494856 moved_to_replicas=144593 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[32776, 33096, 32793, 32438, 33096, 33096, 32412, 32437] avg=32768.0000 max=33096 max_rank=1 min=32412 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=192578 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[33096, 33096, 32290, 33096, 33096, 32776, 32262, 32432] avg=32768.0000 max=33096 max_rank=0 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32422 max_load_reduction_pct=0.494856 moved_to_replicas=144593 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[32776, 65494, 32793, 571, 65522, 64963, 14, 11] avg=32768.0000 max=65522 max_rank=4 min=11 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[33096, 33096, 32290, 33096, 33096, 32776, 32262, 32432] avg=32768.0000 max=33096 max_rank=0 min=32262 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32422 max_load_reduction_pct=0.494856 moved_to_replicas=144593 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[32776, 33096, 32793, 32438, 33096, 33096, 32412, 32437] avg=32768.0000 max=33096 max_rank=1 min=32412 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=192578 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[32776, 33096, 32793, 32438, 33096, 33096, 32412, 32437] avg=32768.0000 max=33096 max_rank=1 min=32412 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=192578 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[32776, 65494, 32793, 571, 65522, 64963, 14, 11] avg=32768.0000 max=65522 max_rank=4 min=11 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[32776, 65494, 32793, 571, 65522, 64963, 14, 11] avg=32768.0000 max=65522 max_rank=4 min=11 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[32776, 33096, 32793, 32438, 33096, 33096, 32412, 32437] avg=32768.0000 max=33096 max_rank=1 min=32412 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=192578 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[4849, 32761, 98270, 17, 32450, 32777, 32818, 28202] avg=32768.0000 max=98270 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[32776, 65494, 32793, 571, 65522, 64963, 14, 11] avg=32768.0000 max=65522 max_rank=4 min=11 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[32776, 33096, 32793, 32438, 33096, 33096, 32412, 32437] avg=32768.0000 max=33096 max_rank=1 min=32412 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=192578 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[32776, 65494, 32793, 571, 65522, 64963, 14, 11] avg=32768.0000 max=65522 max_rank=4 min=11 min_rank=7 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[4849, 32761, 98270, 17, 32450, 32777, 32818, 28202] avg=32768.0000 max=98270 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[4849, 32761, 98270, 17, 32450, 32777, 32818, 28202] avg=32768.0000 max=98270 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[32776, 33096, 32793, 32438, 33096, 33096, 32412, 32437] avg=32768.0000 max=33096 max_rank=1 min=32412 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=192578 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[33097, 32761, 33096, 32776, 32450, 32777, 32818, 32369] avg=32768.0000 max=33097 max_rank=0 min=32369 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65173 max_load_reduction_pct=0.663203 moved_to_replicas=130373 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[32776, 33096, 32793, 32438, 33096, 33096, 32412, 32437] avg=32768.0000 max=33096 max_rank=1 min=32412 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=192578 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[4849, 32761, 98270, 17, 32450, 32777, 32818, 28202] avg=32768.0000 max=98270 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[32776, 33096, 32793, 32438, 33096, 33096, 32412, 32437] avg=32768.0000 max=33096 max_rank=1 min=32412 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=192578 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[33097, 32761, 33096, 32776, 32450, 32777, 32818, 32369] avg=32768.0000 max=33097 max_rank=0 min=32369 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65173 max_load_reduction_pct=0.663203 moved_to_replicas=130373 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[33097, 32761, 33096, 32776, 32450, 32777, 32818, 32369] avg=32768.0000 max=33097 max_rank=0 min=32369 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65173 max_load_reduction_pct=0.663203 moved_to_replicas=130373 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[4849, 32761, 98270, 17, 32450, 32777, 32818, 28202] avg=32768.0000 max=98270 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[4849, 32761, 98270, 17, 32450, 32777, 32818, 28202] avg=32768.0000 max=98270 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[33097, 32761, 33096, 32776, 32450, 32777, 32818, 32369] avg=32768.0000 max=33097 max_rank=0 min=32369 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65173 max_load_reduction_pct=0.663203 moved_to_replicas=130373 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32774, 32755, 32722, 34, 44748, 65513, 20824] avg=32768.0000 max=65513 max_rank=6 min=34 min_rank=4 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[4849, 32761, 98270, 17, 32450, 32777, 32818, 28202] avg=32768.0000 max=98270 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[33097, 32761, 33096, 32776, 32450, 32777, 32818, 32369] avg=32768.0000 max=33097 max_rank=0 min=32369 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65173 max_load_reduction_pct=0.663203 moved_to_replicas=130373 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[4849, 32761, 98270, 17, 32450, 32777, 32818, 28202] avg=32768.0000 max=98270 max_rank=2 min=17 min_rank=3 imbalance_ratio=2.998962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32774, 32755, 32722, 34, 44748, 65513, 20824] avg=32768.0000 max=65513 max_rank=6 min=34 min_rank=4 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32774, 32755, 32722, 34, 44748, 65513, 20824] avg=32768.0000 max=65513 max_rank=6 min=34 min_rank=4 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[33097, 32761, 33096, 32776, 32450, 32777, 32818, 32369] avg=32768.0000 max=33097 max_rank=0 min=32369 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65173 max_load_reduction_pct=0.663203 moved_to_replicas=130373 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32774, 32755, 32722, 32451, 33094, 33096, 32478] avg=32768.0000 max=33096 max_rank=6 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=84969 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[33097, 32761, 33096, 32776, 32450, 32777, 32818, 32369] avg=32768.0000 max=33097 max_rank=0 min=32369 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65173 max_load_reduction_pct=0.663203 moved_to_replicas=130373 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[33097, 32761, 33096, 32776, 32450, 32777, 32818, 32369] avg=32768.0000 max=33097 max_rank=0 min=32369 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=65173 max_load_reduction_pct=0.663203 moved_to_replicas=130373 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32774, 32755, 32722, 34, 44748, 65513, 20824] avg=32768.0000 max=65513 max_rank=6 min=34 min_rank=4 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32774, 32755, 32722, 32451, 33094, 33096, 32478] avg=32768.0000 max=33096 max_rank=6 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=84969 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32774, 32755, 32722, 32451, 33094, 33096, 32478] avg=32768.0000 max=33096 max_rank=6 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=84969 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32774, 32755, 32722, 34, 44748, 65513, 20824] avg=32768.0000 max=65513 max_rank=6 min=34 min_rank=4 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32774, 32755, 32722, 34, 44748, 65513, 20824] avg=32768.0000 max=65513 max_rank=6 min=34 min_rank=4 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[65506, 6, 21, 32773, 65528, 32767, 65524, 19] avg=32768.0000 max=65528 max_rank=4 min=6 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32774, 32755, 32722, 32451, 33094, 33096, 32478] avg=32768.0000 max=33096 max_rank=6 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=84969 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32774, 32755, 32722, 34, 44748, 65513, 20824] avg=32768.0000 max=65513 max_rank=6 min=34 min_rank=4 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32774, 32755, 32722, 32451, 33094, 33096, 32478] avg=32768.0000 max=33096 max_rank=6 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=84969 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32774, 32755, 32722, 34, 44748, 65513, 20824] avg=32768.0000 max=65513 max_rank=6 min=34 min_rank=4 imbalance_ratio=1.999298 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[65506, 6, 21, 32773, 65528, 32767, 65524, 19] avg=32768.0000 max=65528 max_rank=4 min=6 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[65506, 6, 21, 32773, 65528, 32767, 65524, 19] avg=32768.0000 max=65528 max_rank=4 min=6 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[33096, 32438, 32431, 32773, 33096, 32767, 33096, 32447] avg=32768.0000 max=33096 max_rank=0 min=32431 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194590 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32774, 32755, 32722, 32451, 33094, 33096, 32478] avg=32768.0000 max=33096 max_rank=6 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=84969 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32774, 32755, 32722, 32451, 33094, 33096, 32478] avg=32768.0000 max=33096 max_rank=6 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=84969 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[32774, 32774, 32755, 32722, 32451, 33094, 33096, 32478] avg=32768.0000 max=33096 max_rank=6 min=32451 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32417 max_load_reduction_pct=0.494818 moved_to_replicas=84969 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[65506, 6, 21, 32773, 65528, 32767, 65524, 19] avg=32768.0000 max=65528 max_rank=4 min=6 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[33096, 32438, 32431, 32773, 33096, 32767, 33096, 32447] avg=32768.0000 max=33096 max_rank=0 min=32431 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194590 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[65506, 6, 21, 32773, 65528, 32767, 65524, 19] avg=32768.0000 max=65528 max_rank=4 min=6 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[33096, 32438, 32431, 32773, 33096, 32767, 33096, 32447] avg=32768.0000 max=33096 max_rank=0 min=32431 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194590 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[51083, 33025, 65522, 46906, 60, 32768, 20, 32760] avg=32768.0000 max=65522 max_rank=2 min=20 min_rank=6 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[33096, 32438, 32431, 32773, 33096, 32767, 33096, 32447] avg=32768.0000 max=33096 max_rank=0 min=32431 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194590 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[65506, 6, 21, 32773, 65528, 32767, 65524, 19] avg=32768.0000 max=65528 max_rank=4 min=6 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[65506, 6, 21, 32773, 65528, 32767, 65524, 19] avg=32768.0000 max=65528 max_rank=4 min=6 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[33096, 32438, 32431, 32773, 33096, 32767, 33096, 32447] avg=32768.0000 max=33096 max_rank=0 min=32431 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194590 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[65506, 6, 21, 32773, 65528, 32767, 65524, 19] avg=32768.0000 max=65528 max_rank=4 min=6 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[51083, 33025, 65522, 46906, 60, 32768, 20, 32760] avg=32768.0000 max=65522 max_rank=2 min=20 min_rank=6 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[33096, 33025, 33096, 33096, 31857, 32768, 32446, 32760] avg=32768.0000 max=33096 max_rank=0 min=31857 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128617 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[51083, 33025, 65522, 46906, 60, 32768, 20, 32760] avg=32768.0000 max=65522 max_rank=2 min=20 min_rank=6 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[33096, 32438, 32431, 32773, 33096, 32767, 33096, 32447] avg=32768.0000 max=33096 max_rank=0 min=32431 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194590 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[33096, 32438, 32431, 32773, 33096, 32767, 33096, 32447] avg=32768.0000 max=33096 max_rank=0 min=32431 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194590 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[33096, 32438, 32431, 32773, 33096, 32767, 33096, 32447] avg=32768.0000 max=33096 max_rank=0 min=32431 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194590 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[51083, 33025, 65522, 46906, 60, 32768, 20, 32760] avg=32768.0000 max=65522 max_rank=2 min=20 min_rank=6 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[33096, 33025, 33096, 33096, 31857, 32768, 32446, 32760] avg=32768.0000 max=33096 max_rank=0 min=31857 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128617 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[51083, 33025, 65522, 46906, 60, 32768, 20, 32760] avg=32768.0000 max=65522 max_rank=2 min=20 min_rank=6 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[33096, 33025, 33096, 33096, 31857, 32768, 32446, 32760] avg=32768.0000 max=33096 max_rank=0 min=31857 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128617 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[52, 65287, 32778, 32397, 33098, 65467, 32971, 94] avg=32768.0000 max=65467 max_rank=5 min=52 min_rank=0 imbalance_ratio=1.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[33096, 33025, 33096, 33096, 31857, 32768, 32446, 32760] avg=32768.0000 max=33096 max_rank=0 min=31857 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128617 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[51083, 33025, 65522, 46906, 60, 32768, 20, 32760] avg=32768.0000 max=65522 max_rank=2 min=20 min_rank=6 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[51083, 33025, 65522, 46906, 60, 32768, 20, 32760] avg=32768.0000 max=65522 max_rank=2 min=20 min_rank=6 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[33096, 33025, 33096, 33096, 31857, 32768, 32446, 32760] avg=32768.0000 max=33096 max_rank=0 min=31857 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128617 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[51083, 33025, 65522, 46906, 60, 32768, 20, 32760] avg=32768.0000 max=65522 max_rank=2 min=20 min_rank=6 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[52, 65287, 32778, 32397, 33098, 65467, 32971, 94] avg=32768.0000 max=65467 max_rank=5 min=52 min_rank=0 imbalance_ratio=1.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[32423, 33096, 32778, 32397, 33096, 33096, 32971, 32287] avg=32768.0000 max=33096 max_rank=1 min=32287 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32371 max_load_reduction_pct=0.494463 moved_to_replicas=129439 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[52, 65287, 32778, 32397, 33098, 65467, 32971, 94] avg=32768.0000 max=65467 max_rank=5 min=52 min_rank=0 imbalance_ratio=1.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[33096, 33025, 33096, 33096, 31857, 32768, 32446, 32760] avg=32768.0000 max=33096 max_rank=0 min=31857 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128617 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[33096, 33025, 33096, 33096, 31857, 32768, 32446, 32760] avg=32768.0000 max=33096 max_rank=0 min=31857 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128617 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[33096, 33025, 33096, 33096, 31857, 32768, 32446, 32760] avg=32768.0000 max=33096 max_rank=0 min=31857 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128617 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[52, 65287, 32778, 32397, 33098, 65467, 32971, 94] avg=32768.0000 max=65467 max_rank=5 min=52 min_rank=0 imbalance_ratio=1.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[32423, 33096, 32778, 32397, 33096, 33096, 32971, 32287] avg=32768.0000 max=33096 max_rank=1 min=32287 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32371 max_load_reduction_pct=0.494463 moved_to_replicas=129439 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[52, 65287, 32778, 32397, 33098, 65467, 32971, 94] avg=32768.0000 max=65467 max_rank=5 min=52 min_rank=0 imbalance_ratio=1.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[32423, 33096, 32778, 32397, 33096, 33096, 32971, 32287] avg=32768.0000 max=33096 max_rank=1 min=32287 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32371 max_load_reduction_pct=0.494463 moved_to_replicas=129439 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[14, 77, 65552, 49, 65451, 32781, 32759, 65461] avg=32768.0000 max=65552 max_rank=2 min=14 min_rank=0 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[32423, 33096, 32778, 32397, 33096, 33096, 32971, 32287] avg=32768.0000 max=33096 max_rank=1 min=32287 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32371 max_load_reduction_pct=0.494463 moved_to_replicas=129439 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[52, 65287, 32778, 32397, 33098, 65467, 32971, 94] avg=32768.0000 max=65467 max_rank=5 min=52 min_rank=0 imbalance_ratio=1.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[52, 65287, 32778, 32397, 33098, 65467, 32971, 94] avg=32768.0000 max=65467 max_rank=5 min=52 min_rank=0 imbalance_ratio=1.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[32423, 33096, 32778, 32397, 33096, 33096, 32971, 32287] avg=32768.0000 max=33096 max_rank=1 min=32287 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32371 max_load_reduction_pct=0.494463 moved_to_replicas=129439 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[52, 65287, 32778, 32397, 33098, 65467, 32971, 94] avg=32768.0000 max=65467 max_rank=5 min=52 min_rank=0 imbalance_ratio=1.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[32470, 32432, 33096, 32414, 33096, 32781, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=32414 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32456 max_load_reduction_pct=0.495118 moved_to_replicas=194485 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[14, 77, 65552, 49, 65451, 32781, 32759, 65461] avg=32768.0000 max=65552 max_rank=2 min=14 min_rank=0 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[14, 77, 65552, 49, 65451, 32781, 32759, 65461] avg=32768.0000 max=65552 max_rank=2 min=14 min_rank=0 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[32423, 33096, 32778, 32397, 33096, 33096, 32971, 32287] avg=32768.0000 max=33096 max_rank=1 min=32287 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32371 max_load_reduction_pct=0.494463 moved_to_replicas=129439 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[32423, 33096, 32778, 32397, 33096, 33096, 32971, 32287] avg=32768.0000 max=33096 max_rank=1 min=32287 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32371 max_load_reduction_pct=0.494463 moved_to_replicas=129439 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[32423, 33096, 32778, 32397, 33096, 33096, 32971, 32287] avg=32768.0000 max=33096 max_rank=1 min=32287 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32371 max_load_reduction_pct=0.494463 moved_to_replicas=129439 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[14, 77, 65552, 49, 65451, 32781, 32759, 65461] avg=32768.0000 max=65552 max_rank=2 min=14 min_rank=0 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[32470, 32432, 33096, 32414, 33096, 32781, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=32414 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32456 max_load_reduction_pct=0.495118 moved_to_replicas=194485 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[14, 77, 65552, 49, 65451, 32781, 32759, 65461] avg=32768.0000 max=65552 max_rank=2 min=14 min_rank=0 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[32470, 32432, 33096, 32414, 33096, 32781, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=32414 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32456 max_load_reduction_pct=0.495118 moved_to_replicas=194485 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[32780, 32756, 32782, 32778, 163, 65450, 65405, 30] avg=32768.0000 max=65450 max_rank=5 min=30 min_rank=7 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[32470, 32432, 33096, 32414, 33096, 32781, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=32414 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32456 max_load_reduction_pct=0.495118 moved_to_replicas=194485 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[14, 77, 65552, 49, 65451, 32781, 32759, 65461] avg=32768.0000 max=65552 max_rank=2 min=14 min_rank=0 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[14, 77, 65552, 49, 65451, 32781, 32759, 65461] avg=32768.0000 max=65552 max_rank=2 min=14 min_rank=0 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[32470, 32432, 33096, 32414, 33096, 32781, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=32414 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32456 max_load_reduction_pct=0.495118 moved_to_replicas=194485 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[32780, 32756, 32782, 32778, 32472, 33096, 33096, 32384] avg=32768.0000 max=33096 max_rank=5 min=32384 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=129483 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[14, 77, 65552, 49, 65451, 32781, 32759, 65461] avg=32768.0000 max=65552 max_rank=2 min=14 min_rank=0 imbalance_ratio=2.000488 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[32780, 32756, 32782, 32778, 163, 65450, 65405, 30] avg=32768.0000 max=65450 max_rank=5 min=30 min_rank=7 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[32780, 32756, 32782, 32778, 163, 65450, 65405, 30] avg=32768.0000 max=65450 max_rank=5 min=30 min_rank=7 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[32470, 32432, 33096, 32414, 33096, 32781, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=32414 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32456 max_load_reduction_pct=0.495118 moved_to_replicas=194485 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[32470, 32432, 33096, 32414, 33096, 32781, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=32414 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32456 max_load_reduction_pct=0.495118 moved_to_replicas=194485 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[32470, 32432, 33096, 32414, 33096, 32781, 32759, 33096] avg=32768.0000 max=33096 max_rank=2 min=32414 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32456 max_load_reduction_pct=0.495118 moved_to_replicas=194485 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[32780, 32756, 32782, 32778, 163, 65450, 65405, 30] avg=32768.0000 max=65450 max_rank=5 min=30 min_rank=7 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[32780, 32756, 32782, 32778, 32472, 33096, 33096, 32384] avg=32768.0000 max=33096 max_rank=5 min=32384 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=129483 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[32780, 32756, 32782, 32778, 32472, 33096, 33096, 32384] avg=32768.0000 max=33096 max_rank=5 min=32384 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=129483 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[67676, 32836, 33042, 32758, 22, 42, 32767, 63001] avg=32768.0000 max=67676 max_rank=0 min=22 min_rank=4 imbalance_ratio=2.065308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[32780, 32756, 32782, 32778, 163, 65450, 65405, 30] avg=32768.0000 max=65450 max_rank=5 min=30 min_rank=7 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[32780, 32756, 32782, 32778, 32472, 33096, 33096, 32384] avg=32768.0000 max=33096 max_rank=5 min=32384 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=129483 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[32780, 32756, 32782, 32778, 163, 65450, 65405, 30] avg=32768.0000 max=65450 max_rank=5 min=30 min_rank=7 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[32780, 32756, 32782, 32778, 163, 65450, 65405, 30] avg=32768.0000 max=65450 max_rank=5 min=30 min_rank=7 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32836, 33042, 32758, 32767, 31782, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34580 max_load_reduction_pct=0.510964 moved_to_replicas=129033 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[32780, 32756, 32782, 32778, 163, 65450, 65405, 30] avg=32768.0000 max=65450 max_rank=5 min=30 min_rank=7 imbalance_ratio=1.997375 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[32780, 32756, 32782, 32778, 32472, 33096, 33096, 32384] avg=32768.0000 max=33096 max_rank=5 min=32384 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=129483 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[67676, 32836, 33042, 32758, 22, 42, 32767, 63001] avg=32768.0000 max=67676 max_rank=0 min=22 min_rank=4 imbalance_ratio=2.065308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[67676, 32836, 33042, 32758, 22, 42, 32767, 63001] avg=32768.0000 max=67676 max_rank=0 min=22 min_rank=4 imbalance_ratio=2.065308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[32780, 32756, 32782, 32778, 32472, 33096, 33096, 32384] avg=32768.0000 max=33096 max_rank=5 min=32384 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=129483 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[32780, 32756, 32782, 32778, 32472, 33096, 33096, 32384] avg=32768.0000 max=33096 max_rank=5 min=32384 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=129483 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[32780, 32756, 32782, 32778, 32472, 33096, 33096, 32384] avg=32768.0000 max=33096 max_rank=5 min=32384 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32354 max_load_reduction_pct=0.494332 moved_to_replicas=129483 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[67676, 32836, 33042, 32758, 22, 42, 32767, 63001] avg=32768.0000 max=67676 max_rank=0 min=22 min_rank=4 imbalance_ratio=2.065308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32836, 33042, 32758, 32767, 31782, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34580 max_load_reduction_pct=0.510964 moved_to_replicas=129033 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32836, 33042, 32758, 32767, 31782, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34580 max_load_reduction_pct=0.510964 moved_to_replicas=129033 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[67676, 32836, 33042, 32758, 22, 42, 32767, 63001] avg=32768.0000 max=67676 max_rank=0 min=22 min_rank=4 imbalance_ratio=2.065308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[67676, 32836, 33042, 32758, 22, 42, 32767, 63001] avg=32768.0000 max=67676 max_rank=0 min=22 min_rank=4 imbalance_ratio=2.065308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32836, 33042, 32758, 32767, 31782, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34580 max_load_reduction_pct=0.510964 moved_to_replicas=129033 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[67676, 32836, 33042, 32758, 22, 42, 32767, 63001] avg=32768.0000 max=67676 max_rank=0 min=22 min_rank=4 imbalance_ratio=2.065308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[67676, 32836, 33042, 32758, 22, 42, 32767, 63001] avg=32768.0000 max=67676 max_rank=0 min=22 min_rank=4 imbalance_ratio=2.065308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32836, 33042, 32758, 32767, 31782, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34580 max_load_reduction_pct=0.510964 moved_to_replicas=129033 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32836, 33042, 32758, 32767, 31782, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34580 max_load_reduction_pct=0.510964 moved_to_replicas=129033 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32836, 33042, 32758, 32767, 31782, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34580 max_load_reduction_pct=0.510964 moved_to_replicas=129033 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[33096, 32836, 33042, 32758, 32767, 31782, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31782 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=34580 max_load_reduction_pct=0.510964 moved_to_replicas=129033 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=32 total_expected_recv=1049556 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253568 peak_demand=33099 required_capacity=1.010101 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=32 total_expected_recv=1049817 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=252800 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=32 total_expected_recv=1047053 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=256896 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=32 total_expected_recv=1048852 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=255360 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=32 total_expected_recv=1048859 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=255104 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=32 total_expected_recv=1047080 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=256256 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=32 total_expected_recv=1050586 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=252288 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=32 total_expected_recv=1046805 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=255488 peak_demand=33096 required_capacity=1.010010 + [2026-08-30 05:10:30] iteration 8/ 12 | consumed samples: 128 | elapsed time per iteration (ms): 534.7 | throughput per GPU (TFLOP/s/GPU): 623.4 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.557535E+00 | loss scale: 1.0 | grad norm: 5.844 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=40960 expected_recv=65120 expected_overflow=0 exact_recv_per_rank=[32948, 30620, 33095, 33096, 33097, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32948, 30620, 33095, 33096, 33097, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32948, 30620, 33095, 33096, 33097, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=40960 expected_recv=62017 expected_overflow=0 exact_recv_per_rank=[32948, 30620, 33095, 33096, 33097, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32948, 30620, 33095, 33096, 33097, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32948, 30620, 33095, 33096, 33097, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=40960 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[32948, 30620, 33095, 33096, 33097, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=2 budget=40960 expected_recv=65120 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=2 budget=40960 expected_recv=62017 actual_valid_recv=62848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32948, 30620, 33095, 33096, 33097, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=62848 slack_slots=19072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=2 budget=40960 expected_recv=66190 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=40960 expected_recv=64292 expected_overflow=0 exact_recv_per_rank=[32198, 32734, 33096, 33096, 32755, 32770, 32615, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65527 expected_overflow=0 exact_recv_per_rank=[32198, 32734, 33096, 33096, 32755, 32770, 32615, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32198, 32734, 33096, 33096, 32755, 32770, 32615, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32198, 32734, 33096, 33096, 32755, 32770, 32615, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65496 expected_overflow=0 exact_recv_per_rank=[32198, 32734, 33096, 33096, 32755, 32770, 32615, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=2 budget=40960 expected_recv=64292 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65655 expected_overflow=0 exact_recv_per_rank=[32198, 32734, 33096, 33096, 32755, 32770, 32615, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65527 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65418 expected_overflow=0 exact_recv_per_rank=[32198, 32734, 33096, 33096, 32755, 32770, 32615, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65496 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65655 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65418 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65516 expected_overflow=0 exact_recv_per_rank=[32198, 32734, 33096, 33096, 32755, 32770, 32615, 32880] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=2 budget=40960 expected_recv=65516 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65637 expected_overflow=0 exact_recv_per_rank=[32807, 32751, 33096, 32764, 32402, 33096, 32430, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32807, 32751, 33096, 32764, 32402, 33096, 32430, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[32807, 32751, 33096, 32764, 32402, 33096, 32430, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65637 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32807, 32751, 33096, 32764, 32402, 33096, 32430, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65518 expected_overflow=0 exact_recv_per_rank=[32807, 32751, 33096, 32764, 32402, 33096, 32430, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=40960 expected_recv=64820 expected_overflow=0 exact_recv_per_rank=[32807, 32751, 33096, 32764, 32402, 33096, 32430, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65570 expected_overflow=0 exact_recv_per_rank=[32807, 32751, 33096, 32764, 32402, 33096, 32430, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65530 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65518 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=2 budget=40960 expected_recv=64820 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=2 budget=40960 expected_recv=65570 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=40960 expected_recv=64829 expected_overflow=0 exact_recv_per_rank=[32807, 32751, 33096, 32764, 32402, 33096, 32430, 32798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=2 budget=40960 expected_recv=64829 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=40960 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[32770, 33095, 33096, 33096, 33096, 31073, 33096, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=40960 expected_recv=62185 expected_overflow=0 exact_recv_per_rank=[32770, 33095, 33096, 33096, 33096, 31073, 33096, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=2 budget=40960 expected_recv=65538 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=2 budget=40960 expected_recv=62185 actual_valid_recv=62592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=62592 slack_slots=19328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32770, 33095, 33096, 33096, 33096, 31073, 33096, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=40960 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[32770, 33095, 33096, 33096, 33096, 31073, 33096, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32770, 33095, 33096, 33096, 33096, 31073, 33096, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32770, 33095, 33096, 33096, 33096, 31073, 33096, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=40960 expected_recv=65607 expected_overflow=0 exact_recv_per_rank=[32770, 33095, 33096, 33096, 33096, 31073, 33096, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=2 budget=40960 expected_recv=66190 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32770, 33095, 33096, 33096, 33096, 31073, 33096, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=2 budget=40960 expected_recv=65607 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32815, 32772, 32074, 33096, 32782, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32815, 32772, 32074, 33096, 32782, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65542 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32815, 32772, 32074, 33096, 32782, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32815, 32772, 32074, 33096, 32782, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65624 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32815, 32772, 32074, 33096, 32782, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65566 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32815, 32772, 32074, 33096, 32782, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65462 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32815, 32772, 32074, 33096, 32782, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65541 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65624 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65566 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=40960 expected_recv=64169 expected_overflow=0 exact_recv_per_rank=[32774, 33096, 32815, 32772, 32074, 33096, 32782, 32735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=2 budget=40960 expected_recv=65462 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=2 budget=40960 expected_recv=64169 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33094, 31784, 32909, 32728, 33096, 33096, 32342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 33094, 31784, 32909, 32728, 33096, 33096, 32342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=40960 expected_recv=66006 expected_overflow=0 exact_recv_per_rank=[33095, 33094, 31784, 32909, 32728, 33096, 33096, 32342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=40960 expected_recv=66189 expected_overflow=0 exact_recv_per_rank=[33095, 33094, 31784, 32909, 32728, 33096, 33096, 32342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=40960 expected_recv=63379 expected_overflow=0 exact_recv_per_rank=[33095, 33094, 31784, 32909, 32728, 33096, 33096, 32342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33094, 31784, 32909, 32728, 33096, 33096, 32342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=40960 expected_recv=64677 expected_overflow=0 exact_recv_per_rank=[33095, 33094, 31784, 32909, 32728, 33096, 33096, 32342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=2 budget=40960 expected_recv=66006 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=2 budget=40960 expected_recv=66189 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=2 budget=40960 expected_recv=63379 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=40960 expected_recv=65462 expected_overflow=0 exact_recv_per_rank=[33095, 33094, 31784, 32909, 32728, 33096, 33096, 32342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=2 budget=40960 expected_recv=64677 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=40960 expected_recv=65516 expected_overflow=0 exact_recv_per_rank=[33095, 33025, 32760, 33096, 33096, 32756, 32459, 31857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=2 budget=40960 expected_recv=65462 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=40960 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[33095, 33025, 32760, 33096, 33096, 32756, 32459, 31857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=2 budget=40960 expected_recv=65516 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=2 budget=40960 expected_recv=66190 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33025, 32760, 33096, 33096, 32756, 32459, 31857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=40960 expected_recv=66036 expected_overflow=0 exact_recv_per_rank=[33095, 33025, 32760, 33096, 33096, 32756, 32459, 31857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=40960 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[33095, 33025, 32760, 33096, 33096, 32756, 32459, 31857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=2 budget=40960 expected_recv=66036 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=2 budget=40960 expected_recv=65526 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=40960 expected_recv=64911 expected_overflow=0 exact_recv_per_rank=[33095, 33025, 32760, 33096, 33096, 32756, 32459, 31857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=40960 expected_recv=63725 expected_overflow=0 exact_recv_per_rank=[33095, 33025, 32760, 33096, 33096, 32756, 32459, 31857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=2 budget=40960 expected_recv=64911 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=2 budget=40960 expected_recv=63725 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33025, 32760, 33096, 33096, 32756, 32459, 31857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65524 expected_overflow=0 exact_recv_per_rank=[32768, 32762, 33096, 33096, 33096, 32760, 32768, 31798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65524 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[32768, 32762, 33096, 33096, 33096, 32760, 32768, 31798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65534 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=40960 expected_recv=66184 expected_overflow=0 exact_recv_per_rank=[32768, 32762, 33096, 33096, 33096, 32760, 32768, 31798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65525 expected_overflow=0 exact_recv_per_rank=[32768, 32762, 33096, 33096, 33096, 32760, 32768, 31798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=2 budget=40960 expected_recv=66184 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32768, 32762, 33096, 33096, 33096, 32760, 32768, 31798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65525 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[32768, 32762, 33096, 33096, 33096, 32760, 32768, 31798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=40960 expected_recv=63607 expected_overflow=0 exact_recv_per_rank=[32768, 32762, 33096, 33096, 33096, 32760, 32768, 31798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32768, 32762, 33096, 33096, 33096, 32760, 32768, 31798] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[31886, 32764, 33096, 32772, 32663, 32771, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=2 budget=40960 expected_recv=65530 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=40960 expected_recv=63860 expected_overflow=0 exact_recv_per_rank=[31886, 32764, 33096, 32772, 32663, 32771, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=2 budget=40960 expected_recv=63607 actual_valid_recv=64384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65540 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=64384 slack_slots=17536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=2 budget=40960 expected_recv=63860 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[31886, 32764, 33096, 32772, 32663, 32771, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[31886, 32764, 33096, 32772, 32663, 32771, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31886, 32764, 33096, 32772, 32663, 32771, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65540 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65532 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=40960 expected_recv=66189 expected_overflow=0 exact_recv_per_rank=[31886, 32764, 33096, 32772, 32663, 32771, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=40960 expected_recv=66195 expected_overflow=0 exact_recv_per_rank=[31886, 32764, 33096, 32772, 32663, 32771, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65240 expected_overflow=0 exact_recv_per_rank=[31886, 32764, 33096, 32772, 32663, 32771, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33094, 32763, 32656, 32285, 32058, 33098, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=2 budget=40960 expected_recv=66189 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=2 budget=40960 expected_recv=66195 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=2 budget=40960 expected_recv=65240 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=40960 expected_recv=66189 expected_overflow=0 exact_recv_per_rank=[33094, 32763, 32656, 32285, 32058, 33098, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=2 budget=40960 expected_recv=66189 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=40960 expected_recv=64619 expected_overflow=0 exact_recv_per_rank=[33094, 32763, 32656, 32285, 32058, 33098, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=40960 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[33094, 32763, 32656, 32285, 32058, 33098, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=2 budget=40960 expected_recv=64619 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=40960 expected_recv=65245 expected_overflow=0 exact_recv_per_rank=[33094, 32763, 32656, 32285, 32058, 33098, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=2 budget=40960 expected_recv=65526 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=2 budget=40960 expected_recv=65245 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=40960 expected_recv=64135 expected_overflow=0 exact_recv_per_rank=[33094, 32763, 32656, 32285, 32058, 33098, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33094, 32763, 32656, 32285, 32058, 33098, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=40960 expected_recv=66188 expected_overflow=0 exact_recv_per_rank=[33094, 32763, 32656, 32285, 32058, 33098, 33096, 33094] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=40960 expected_recv=65506 expected_overflow=0 exact_recv_per_rank=[33096, 32436, 32426, 32467, 33096, 32775, 33096, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=2 budget=40960 expected_recv=64135 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=2 budget=40960 expected_recv=66188 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32436, 32426, 32467, 33096, 32775, 33096, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=2 budget=40960 expected_recv=65506 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=40960 expected_recv=64963 expected_overflow=0 exact_recv_per_rank=[33096, 32436, 32426, 32467, 33096, 32775, 33096, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=40960 expected_recv=64879 expected_overflow=0 exact_recv_per_rank=[33096, 32436, 32426, 32467, 33096, 32775, 33096, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=2 budget=40960 expected_recv=64963 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=40960 expected_recv=64868 expected_overflow=0 exact_recv_per_rank=[33096, 32436, 32426, 32467, 33096, 32775, 33096, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=2 budget=40960 expected_recv=64879 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=2 budget=40960 expected_recv=64868 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32436, 32426, 32467, 33096, 32775, 33096, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32436, 32426, 32467, 33096, 32775, 33096, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=40960 expected_recv=65512 expected_overflow=0 exact_recv_per_rank=[32440, 33096, 33096, 33096, 32449, 32753, 32452, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=40960 expected_recv=65496 expected_overflow=0 exact_recv_per_rank=[33096, 32436, 32426, 32467, 33096, 32775, 33096, 32752] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=40960 expected_recv=64894 expected_overflow=0 exact_recv_per_rank=[32440, 33096, 33096, 33096, 32449, 32753, 32452, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=2 budget=40960 expected_recv=65512 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=2 budget=40960 expected_recv=65496 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=2 budget=40960 expected_recv=64894 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32440, 33096, 33096, 33096, 32449, 32753, 32452, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32440, 33096, 33096, 33096, 32449, 32753, 32452, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32440, 33096, 33096, 33096, 32449, 32753, 32452, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=40960 expected_recv=64883 expected_overflow=0 exact_recv_per_rank=[32440, 33096, 33096, 33096, 32449, 32753, 32452, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=40960 expected_recv=64894 expected_overflow=0 exact_recv_per_rank=[32440, 33096, 33096, 33096, 32449, 32753, 32452, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=40960 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[32440, 33096, 33096, 33096, 32449, 32753, 32452, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=2 budget=40960 expected_recv=64883 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32464, 32812, 33096, 32118, 32750, 33096, 33055, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=2 budget=40960 expected_recv=64894 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=2 budget=40960 expected_recv=65529 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=40960 expected_recv=64907 expected_overflow=0 exact_recv_per_rank=[32464, 32812, 33096, 32118, 32750, 33096, 33055, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=12 calls=2 budget=40960 expected_recv=64907 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=40960 expected_recv=64440 expected_overflow=0 exact_recv_per_rank=[32464, 32812, 33096, 32118, 32750, 33096, 33055, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65586 expected_overflow=0 exact_recv_per_rank=[32464, 32812, 33096, 32118, 32750, 33096, 33055, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32464, 32812, 33096, 32118, 32750, 33096, 33055, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=2 budget=40960 expected_recv=64440 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65586 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65504 expected_overflow=0 exact_recv_per_rank=[32464, 32812, 33096, 32118, 32750, 33096, 33055, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65947 expected_overflow=0 exact_recv_per_rank=[32464, 32812, 33096, 32118, 32750, 33096, 33055, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65520 expected_overflow=0 exact_recv_per_rank=[32464, 32812, 33096, 32118, 32750, 33096, 33055, 32753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65547 expected_overflow=0 exact_recv_per_rank=[32248, 32552, 33096, 32766, 33096, 32779, 32764, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65504 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65947 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=12 calls=2 budget=40960 expected_recv=65520 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=40960 expected_recv=64651 expected_overflow=0 exact_recv_per_rank=[32248, 32552, 33096, 32766, 33096, 32779, 32764, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65547 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=13 calls=2 budget=40960 expected_recv=64651 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=40960 expected_recv=64996 expected_overflow=0 exact_recv_per_rank=[32248, 32552, 33096, 32766, 33096, 32779, 32764, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[32248, 32552, 33096, 32766, 33096, 32779, 32764, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32248, 32552, 33096, 32766, 33096, 32779, 32764, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=13 calls=2 budget=40960 expected_recv=64996 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65533 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32248, 32552, 33096, 32766, 33096, 32779, 32764, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[32248, 32552, 33096, 32766, 33096, 32779, 32764, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65645 expected_overflow=0 exact_recv_per_rank=[32248, 32552, 33096, 32766, 33096, 32779, 32764, 32843] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[32186, 32765, 32767, 32773, 33096, 32765, 32696, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65645 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65533 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=2 budget=40960 expected_recv=65532 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=40960 expected_recv=64320 expected_overflow=0 exact_recv_per_rank=[32186, 32765, 32767, 32773, 33096, 32765, 32696, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=14 calls=2 budget=40960 expected_recv=64320 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[32186, 32765, 32767, 32773, 33096, 32765, 32696, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[32186, 32765, 32767, 32773, 33096, 32765, 32696, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[32186, 32765, 32767, 32773, 33096, 32765, 32696, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65532 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65532 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65542 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65370 expected_overflow=0 exact_recv_per_rank=[33096, 32943, 33096, 32775, 32117, 32621, 32768, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32186, 32765, 32767, 32773, 33096, 32765, 32696, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32186, 32765, 32767, 32773, 33096, 32765, 32696, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65445 expected_overflow=0 exact_recv_per_rank=[32186, 32765, 32767, 32773, 33096, 32765, 32696, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65370 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32943, 33096, 32775, 32117, 32621, 32768, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=2 budget=40960 expected_recv=65445 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65646 expected_overflow=0 exact_recv_per_rank=[33096, 32943, 33096, 32775, 32117, 32621, 32768, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32943, 33096, 32775, 32117, 32621, 32768, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65542 expected_overflow=0 exact_recv_per_rank=[33096, 32943, 33096, 32775, 32117, 32621, 32768, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65646 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65542 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[26011, 22544, 14873, 25871, 12430, 24929, 55425, 80061] avg=32768.0000 max=80061 max_rank=7 min=12430 min_rank=4 imbalance_ratio=2.443268 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65483 expected_overflow=0 exact_recv_per_rank=[33096, 32943, 33096, 32775, 32117, 32621, 32768, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=40960 expected_recv=64327 expected_overflow=0 exact_recv_per_rank=[33096, 32943, 33096, 32775, 32117, 32621, 32768, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[33096, 32943, 33096, 32775, 32117, 32621, 32768, 32728] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65483 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=15 calls=2 budget=40960 expected_recv=64327 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=15 calls=2 budget=40960 expected_recv=65536 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[26011, 22544, 14873, 25871, 12430, 24929, 55425, 80061] avg=32768.0000 max=80061 max_rank=7 min=12430 min_rank=4 imbalance_ratio=2.443268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[26011, 22544, 14873, 25871, 12430, 24929, 55425, 80061] avg=32768.0000 max=80061 max_rank=7 min=12430 min_rank=4 imbalance_ratio=2.443268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[32948, 30620, 33095, 33096, 33097, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=4 min=30620 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=46964 max_load_reduction_pct=0.586603 moved_to_replicas=141062 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[26011, 22544, 14873, 25871, 12430, 24929, 55425, 80061] avg=32768.0000 max=80061 max_rank=7 min=12430 min_rank=4 imbalance_ratio=2.443268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[26011, 22544, 14873, 25871, 12430, 24929, 55425, 80061] avg=32768.0000 max=80061 max_rank=7 min=12430 min_rank=4 imbalance_ratio=2.443268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[32948, 30620, 33095, 33096, 33097, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=4 min=30620 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=46964 max_load_reduction_pct=0.586603 moved_to_replicas=141062 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[32948, 30620, 33095, 33096, 33097, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=4 min=30620 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=46964 max_load_reduction_pct=0.586603 moved_to_replicas=141062 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[32948, 30620, 33095, 33096, 33097, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=4 min=30620 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=46964 max_load_reduction_pct=0.586603 moved_to_replicas=141062 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[32948, 30620, 33095, 33096, 33097, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=4 min=30620 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=46964 max_load_reduction_pct=0.586603 moved_to_replicas=141062 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[26011, 22544, 14873, 25871, 12430, 24929, 55425, 80061] avg=32768.0000 max=80061 max_rank=7 min=12430 min_rank=4 imbalance_ratio=2.443268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[26011, 22544, 14873, 25871, 12430, 24929, 55425, 80061] avg=32768.0000 max=80061 max_rank=7 min=12430 min_rank=4 imbalance_ratio=2.443268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[26011, 22544, 14873, 25871, 12430, 24929, 55425, 80061] avg=32768.0000 max=80061 max_rank=7 min=12430 min_rank=4 imbalance_ratio=2.443268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[32198, 32734, 66070, 65430, 32755, 6, 71, 32880] avg=32768.0000 max=66070 max_rank=2 min=6 min_rank=5 imbalance_ratio=2.016296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[32198, 32734, 66070, 65430, 32755, 6, 71, 32880] avg=32768.0000 max=66070 max_rank=2 min=6 min_rank=5 imbalance_ratio=2.016296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[32948, 30620, 33095, 33096, 33097, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=4 min=30620 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=46964 max_load_reduction_pct=0.586603 moved_to_replicas=141062 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[32198, 32734, 33096, 33096, 32755, 32770, 32615, 32880] avg=32768.0000 max=33096 max_rank=2 min=32198 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32974 max_load_reduction_pct=0.499077 moved_to_replicas=130854 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[32948, 30620, 33095, 33096, 33097, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=4 min=30620 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=46964 max_load_reduction_pct=0.586603 moved_to_replicas=141062 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[32948, 30620, 33095, 33096, 33097, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=4 min=30620 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=46964 max_load_reduction_pct=0.586603 moved_to_replicas=141062 active_clone_links=15 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[32198, 32734, 66070, 65430, 32755, 6, 71, 32880] avg=32768.0000 max=66070 max_rank=2 min=6 min_rank=5 imbalance_ratio=2.016296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[32198, 32734, 66070, 65430, 32755, 6, 71, 32880] avg=32768.0000 max=66070 max_rank=2 min=6 min_rank=5 imbalance_ratio=2.016296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[32198, 32734, 66070, 65430, 32755, 6, 71, 32880] avg=32768.0000 max=66070 max_rank=2 min=6 min_rank=5 imbalance_ratio=2.016296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[32198, 32734, 33096, 33096, 32755, 32770, 32615, 32880] avg=32768.0000 max=33096 max_rank=2 min=32198 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32974 max_load_reduction_pct=0.499077 moved_to_replicas=130854 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[32198, 32734, 33096, 33096, 32755, 32770, 32615, 32880] avg=32768.0000 max=33096 max_rank=2 min=32198 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32974 max_load_reduction_pct=0.499077 moved_to_replicas=130854 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[32198, 32734, 33096, 33096, 32755, 32770, 32615, 32880] avg=32768.0000 max=33096 max_rank=2 min=32198 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32974 max_load_reduction_pct=0.499077 moved_to_replicas=130854 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[32198, 32734, 33096, 33096, 32755, 32770, 32615, 32880] avg=32768.0000 max=33096 max_rank=2 min=32198 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32974 max_load_reduction_pct=0.499077 moved_to_replicas=130854 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[32198, 32734, 66070, 65430, 32755, 6, 71, 32880] avg=32768.0000 max=66070 max_rank=2 min=6 min_rank=5 imbalance_ratio=2.016296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32751, 34821, 32764, 18, 65480, 30705, 32798] avg=32768.0000 max=65480 max_rank=5 min=18 min_rank=4 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[32198, 32734, 66070, 65430, 32755, 6, 71, 32880] avg=32768.0000 max=66070 max_rank=2 min=6 min_rank=5 imbalance_ratio=2.016296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[32198, 32734, 66070, 65430, 32755, 6, 71, 32880] avg=32768.0000 max=66070 max_rank=2 min=6 min_rank=5 imbalance_ratio=2.016296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32751, 33096, 32764, 32402, 33096, 32430, 32798] avg=32768.0000 max=33096 max_rank=2 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32384 max_load_reduction_pct=0.494563 moved_to_replicas=68598 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32751, 34821, 32764, 18, 65480, 30705, 32798] avg=32768.0000 max=65480 max_rank=5 min=18 min_rank=4 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[32198, 32734, 33096, 33096, 32755, 32770, 32615, 32880] avg=32768.0000 max=33096 max_rank=2 min=32198 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32974 max_load_reduction_pct=0.499077 moved_to_replicas=130854 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[32198, 32734, 33096, 33096, 32755, 32770, 32615, 32880] avg=32768.0000 max=33096 max_rank=2 min=32198 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32974 max_load_reduction_pct=0.499077 moved_to_replicas=130854 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[32198, 32734, 33096, 33096, 32755, 32770, 32615, 32880] avg=32768.0000 max=33096 max_rank=2 min=32198 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32974 max_load_reduction_pct=0.499077 moved_to_replicas=130854 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32751, 34821, 32764, 18, 65480, 30705, 32798] avg=32768.0000 max=65480 max_rank=5 min=18 min_rank=4 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32751, 34821, 32764, 18, 65480, 30705, 32798] avg=32768.0000 max=65480 max_rank=5 min=18 min_rank=4 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32751, 34821, 32764, 18, 65480, 30705, 32798] avg=32768.0000 max=65480 max_rank=5 min=18 min_rank=4 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32751, 33096, 32764, 32402, 33096, 32430, 32798] avg=32768.0000 max=33096 max_rank=2 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32384 max_load_reduction_pct=0.494563 moved_to_replicas=68598 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32751, 33096, 32764, 32402, 33096, 32430, 32798] avg=32768.0000 max=33096 max_rank=2 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32384 max_load_reduction_pct=0.494563 moved_to_replicas=68598 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[8, 6925, 65480, 8, 65517, 22978, 68406, 32822] avg=32768.0000 max=68406 max_rank=6 min=8 min_rank=0 imbalance_ratio=2.087585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32751, 33096, 32764, 32402, 33096, 32430, 32798] avg=32768.0000 max=33096 max_rank=2 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32384 max_load_reduction_pct=0.494563 moved_to_replicas=68598 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32751, 33096, 32764, 32402, 33096, 32430, 32798] avg=32768.0000 max=33096 max_rank=2 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32384 max_load_reduction_pct=0.494563 moved_to_replicas=68598 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32751, 34821, 32764, 18, 65480, 30705, 32798] avg=32768.0000 max=65480 max_rank=5 min=18 min_rank=4 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32751, 34821, 32764, 18, 65480, 30705, 32798] avg=32768.0000 max=65480 max_rank=5 min=18 min_rank=4 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32751, 34821, 32764, 18, 65480, 30705, 32798] avg=32768.0000 max=65480 max_rank=5 min=18 min_rank=4 imbalance_ratio=1.998291 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[32770, 33095, 33096, 33096, 33096, 31073, 33096, 32822] avg=32768.0000 max=33096 max_rank=2 min=31073 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.516183 moved_to_replicas=200196 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[8, 6925, 65480, 8, 65517, 22978, 68406, 32822] avg=32768.0000 max=68406 max_rank=6 min=8 min_rank=0 imbalance_ratio=2.087585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32751, 33096, 32764, 32402, 33096, 32430, 32798] avg=32768.0000 max=33096 max_rank=2 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32384 max_load_reduction_pct=0.494563 moved_to_replicas=68598 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32751, 33096, 32764, 32402, 33096, 32430, 32798] avg=32768.0000 max=33096 max_rank=2 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32384 max_load_reduction_pct=0.494563 moved_to_replicas=68598 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[32807, 32751, 33096, 32764, 32402, 33096, 32430, 32798] avg=32768.0000 max=33096 max_rank=2 min=32402 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32384 max_load_reduction_pct=0.494563 moved_to_replicas=68598 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[8, 6925, 65480, 8, 65517, 22978, 68406, 32822] avg=32768.0000 max=68406 max_rank=6 min=8 min_rank=0 imbalance_ratio=2.087585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[8, 6925, 65480, 8, 65517, 22978, 68406, 32822] avg=32768.0000 max=68406 max_rank=6 min=8 min_rank=0 imbalance_ratio=2.087585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[8, 6925, 65480, 8, 65517, 22978, 68406, 32822] avg=32768.0000 max=68406 max_rank=6 min=8 min_rank=0 imbalance_ratio=2.087585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[32770, 33095, 33096, 33096, 33096, 31073, 33096, 32822] avg=32768.0000 max=33096 max_rank=2 min=31073 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.516183 moved_to_replicas=200196 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[11, 69026, 32815, 32772, 30, 61973, 32782, 32735] avg=32768.0000 max=69026 max_rank=1 min=11 min_rank=0 imbalance_ratio=2.106506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[32770, 33095, 33096, 33096, 33096, 31073, 33096, 32822] avg=32768.0000 max=33096 max_rank=2 min=31073 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.516183 moved_to_replicas=200196 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[32770, 33095, 33096, 33096, 33096, 31073, 33096, 32822] avg=32768.0000 max=33096 max_rank=2 min=31073 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.516183 moved_to_replicas=200196 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[8, 6925, 65480, 8, 65517, 22978, 68406, 32822] avg=32768.0000 max=68406 max_rank=6 min=8 min_rank=0 imbalance_ratio=2.087585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[32770, 33095, 33096, 33096, 33096, 31073, 33096, 32822] avg=32768.0000 max=33096 max_rank=2 min=31073 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.516183 moved_to_replicas=200196 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[8, 6925, 65480, 8, 65517, 22978, 68406, 32822] avg=32768.0000 max=68406 max_rank=6 min=8 min_rank=0 imbalance_ratio=2.087585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[8, 6925, 65480, 8, 65517, 22978, 68406, 32822] avg=32768.0000 max=68406 max_rank=6 min=8 min_rank=0 imbalance_ratio=2.087585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[32774, 33096, 32815, 32772, 32074, 33096, 32782, 32735] avg=32768.0000 max=33096 max_rank=1 min=32074 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35930 max_load_reduction_pct=0.520528 moved_to_replicas=129660 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[11, 69026, 32815, 32772, 30, 61973, 32782, 32735] avg=32768.0000 max=69026 max_rank=1 min=11 min_rank=0 imbalance_ratio=2.106506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[32770, 33095, 33096, 33096, 33096, 31073, 33096, 32822] avg=32768.0000 max=33096 max_rank=2 min=31073 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.516183 moved_to_replicas=200196 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[32770, 33095, 33096, 33096, 33096, 31073, 33096, 32822] avg=32768.0000 max=33096 max_rank=2 min=31073 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.516183 moved_to_replicas=200196 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[11, 69026, 32815, 32772, 30, 61973, 32782, 32735] avg=32768.0000 max=69026 max_rank=1 min=11 min_rank=0 imbalance_ratio=2.106506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[32770, 33095, 33096, 33096, 33096, 31073, 33096, 32822] avg=32768.0000 max=33096 max_rank=2 min=31073 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.516183 moved_to_replicas=200196 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[11, 69026, 32815, 32772, 30, 61973, 32782, 32735] avg=32768.0000 max=69026 max_rank=1 min=11 min_rank=0 imbalance_ratio=2.106506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[11, 69026, 32815, 32772, 30, 61973, 32782, 32735] avg=32768.0000 max=69026 max_rank=1 min=11 min_rank=0 imbalance_ratio=2.106506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[32774, 33096, 32815, 32772, 32074, 33096, 32782, 32735] avg=32768.0000 max=33096 max_rank=1 min=32074 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35930 max_load_reduction_pct=0.520528 moved_to_replicas=129660 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[56049, 42907, 23, 142, 32728, 64857, 65432, 6] avg=32768.0000 max=65432 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.996826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[32774, 33096, 32815, 32772, 32074, 33096, 32782, 32735] avg=32768.0000 max=33096 max_rank=1 min=32074 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35930 max_load_reduction_pct=0.520528 moved_to_replicas=129660 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[32774, 33096, 32815, 32772, 32074, 33096, 32782, 32735] avg=32768.0000 max=33096 max_rank=1 min=32074 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35930 max_load_reduction_pct=0.520528 moved_to_replicas=129660 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[11, 69026, 32815, 32772, 30, 61973, 32782, 32735] avg=32768.0000 max=69026 max_rank=1 min=11 min_rank=0 imbalance_ratio=2.106506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[32774, 33096, 32815, 32772, 32074, 33096, 32782, 32735] avg=32768.0000 max=33096 max_rank=1 min=32074 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35930 max_load_reduction_pct=0.520528 moved_to_replicas=129660 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[33095, 33094, 31784, 32909, 32728, 33096, 33096, 32342] avg=32768.0000 max=33096 max_rank=5 min=31784 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32336 max_load_reduction_pct=0.494192 moved_to_replicas=193644 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[11, 69026, 32815, 32772, 30, 61973, 32782, 32735] avg=32768.0000 max=69026 max_rank=1 min=11 min_rank=0 imbalance_ratio=2.106506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[11, 69026, 32815, 32772, 30, 61973, 32782, 32735] avg=32768.0000 max=69026 max_rank=1 min=11 min_rank=0 imbalance_ratio=2.106506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[56049, 42907, 23, 142, 32728, 64857, 65432, 6] avg=32768.0000 max=65432 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.996826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[32774, 33096, 32815, 32772, 32074, 33096, 32782, 32735] avg=32768.0000 max=33096 max_rank=1 min=32074 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35930 max_load_reduction_pct=0.520528 moved_to_replicas=129660 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[56049, 42907, 23, 142, 32728, 64857, 65432, 6] avg=32768.0000 max=65432 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.996826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[32774, 33096, 32815, 32772, 32074, 33096, 32782, 32735] avg=32768.0000 max=33096 max_rank=1 min=32074 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35930 max_load_reduction_pct=0.520528 moved_to_replicas=129660 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[56049, 42907, 23, 142, 32728, 64857, 65432, 6] avg=32768.0000 max=65432 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.996826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[32774, 33096, 32815, 32772, 32074, 33096, 32782, 32735] avg=32768.0000 max=33096 max_rank=1 min=32074 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35930 max_load_reduction_pct=0.520528 moved_to_replicas=129660 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[43008, 607, 32760, 65536, 65514, 32756, 19, 21944] avg=32768.0000 max=65536 max_rank=3 min=19 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[33095, 33094, 31784, 32909, 32728, 33096, 33096, 32342] avg=32768.0000 max=33096 max_rank=5 min=31784 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32336 max_load_reduction_pct=0.494192 moved_to_replicas=193644 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[56049, 42907, 23, 142, 32728, 64857, 65432, 6] avg=32768.0000 max=65432 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.996826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[33095, 33094, 31784, 32909, 32728, 33096, 33096, 32342] avg=32768.0000 max=33096 max_rank=5 min=31784 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32336 max_load_reduction_pct=0.494192 moved_to_replicas=193644 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[56049, 42907, 23, 142, 32728, 64857, 65432, 6] avg=32768.0000 max=65432 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.996826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[33095, 33094, 31784, 32909, 32728, 33096, 33096, 32342] avg=32768.0000 max=33096 max_rank=5 min=31784 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32336 max_load_reduction_pct=0.494192 moved_to_replicas=193644 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[33095, 33025, 32760, 33096, 33096, 32756, 32459, 31857] avg=32768.0000 max=33096 max_rank=3 min=31857 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=149533 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[33095, 33094, 31784, 32909, 32728, 33096, 33096, 32342] avg=32768.0000 max=33096 max_rank=5 min=31784 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32336 max_load_reduction_pct=0.494192 moved_to_replicas=193644 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[56049, 42907, 23, 142, 32728, 64857, 65432, 6] avg=32768.0000 max=65432 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.996826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[56049, 42907, 23, 142, 32728, 64857, 65432, 6] avg=32768.0000 max=65432 max_rank=6 min=6 min_rank=7 imbalance_ratio=1.996826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[43008, 607, 32760, 65536, 65514, 32756, 19, 21944] avg=32768.0000 max=65536 max_rank=3 min=19 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[33095, 33094, 31784, 32909, 32728, 33096, 33096, 32342] avg=32768.0000 max=33096 max_rank=5 min=31784 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32336 max_load_reduction_pct=0.494192 moved_to_replicas=193644 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[43008, 607, 32760, 65536, 65514, 32756, 19, 21944] avg=32768.0000 max=65536 max_rank=3 min=19 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[33095, 33094, 31784, 32909, 32728, 33096, 33096, 32342] avg=32768.0000 max=33096 max_rank=5 min=31784 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32336 max_load_reduction_pct=0.494192 moved_to_replicas=193644 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[33095, 33094, 31784, 32909, 32728, 33096, 33096, 32342] avg=32768.0000 max=33096 max_rank=5 min=31784 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32336 max_load_reduction_pct=0.494192 moved_to_replicas=193644 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[33095, 33025, 32760, 33096, 33096, 32756, 32459, 31857] avg=32768.0000 max=33096 max_rank=3 min=31857 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=149533 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32762, 97476, 833, 65525, 32760, 6, 14] avg=32768.0000 max=97476 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.974731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[43008, 607, 32760, 65536, 65514, 32756, 19, 21944] avg=32768.0000 max=65536 max_rank=3 min=19 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[43008, 607, 32760, 65536, 65514, 32756, 19, 21944] avg=32768.0000 max=65536 max_rank=3 min=19 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[33095, 33025, 32760, 33096, 33096, 32756, 32459, 31857] avg=32768.0000 max=33096 max_rank=3 min=31857 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=149533 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[43008, 607, 32760, 65536, 65514, 32756, 19, 21944] avg=32768.0000 max=65536 max_rank=3 min=19 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32762, 33096, 33096, 33096, 32760, 32768, 31798] avg=32768.0000 max=33096 max_rank=2 min=31798 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64380 max_load_reduction_pct=0.660470 moved_to_replicas=193815 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[33095, 33025, 32760, 33096, 33096, 32756, 32459, 31857] avg=32768.0000 max=33096 max_rank=3 min=31857 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=149533 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[43008, 607, 32760, 65536, 65514, 32756, 19, 21944] avg=32768.0000 max=65536 max_rank=3 min=19 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[33095, 33025, 32760, 33096, 33096, 32756, 32459, 31857] avg=32768.0000 max=33096 max_rank=3 min=31857 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=149533 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[43008, 607, 32760, 65536, 65514, 32756, 19, 21944] avg=32768.0000 max=65536 max_rank=3 min=19 min_rank=6 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32762, 97476, 833, 65525, 32760, 6, 14] avg=32768.0000 max=97476 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.974731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[33095, 33025, 32760, 33096, 33096, 32756, 32459, 31857] avg=32768.0000 max=33096 max_rank=3 min=31857 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=149533 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32762, 97476, 833, 65525, 32760, 6, 14] avg=32768.0000 max=97476 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.974731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[33095, 33025, 32760, 33096, 33096, 32756, 32459, 31857] avg=32768.0000 max=33096 max_rank=3 min=31857 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=149533 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[33095, 33025, 32760, 33096, 33096, 32756, 32459, 31857] avg=32768.0000 max=33096 max_rank=3 min=31857 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=149533 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32762, 33096, 33096, 33096, 32760, 32768, 31798] avg=32768.0000 max=33096 max_rank=2 min=31798 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64380 max_load_reduction_pct=0.660470 moved_to_replicas=193815 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[28670, 32764, 98284, 10, 237, 32771, 23944, 45464] avg=32768.0000 max=98284 max_rank=2 min=10 min_rank=3 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32762, 97476, 833, 65525, 32760, 6, 14] avg=32768.0000 max=97476 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.974731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32762, 33096, 33096, 33096, 32760, 32768, 31798] avg=32768.0000 max=33096 max_rank=2 min=31798 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64380 max_load_reduction_pct=0.660470 moved_to_replicas=193815 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32762, 97476, 833, 65525, 32760, 6, 14] avg=32768.0000 max=97476 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.974731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32762, 97476, 833, 65525, 32760, 6, 14] avg=32768.0000 max=97476 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.974731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[31886, 32764, 33096, 32772, 32663, 32771, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=31886 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=65186 max_load_reduction_pct=0.663241 moved_to_replicas=155422 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32762, 97476, 833, 65525, 32760, 6, 14] avg=32768.0000 max=97476 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.974731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32762, 33096, 33096, 33096, 32760, 32768, 31798] avg=32768.0000 max=33096 max_rank=2 min=31798 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64380 max_load_reduction_pct=0.660470 moved_to_replicas=193815 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32762, 33096, 33096, 33096, 32760, 32768, 31798] avg=32768.0000 max=33096 max_rank=2 min=31798 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64380 max_load_reduction_pct=0.660470 moved_to_replicas=193815 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32762, 97476, 833, 65525, 32760, 6, 14] avg=32768.0000 max=97476 max_rank=2 min=6 min_rank=6 imbalance_ratio=2.974731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[28670, 32764, 98284, 10, 237, 32771, 23944, 45464] avg=32768.0000 max=98284 max_rank=2 min=10 min_rank=3 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32762, 33096, 33096, 33096, 32760, 32768, 31798] avg=32768.0000 max=33096 max_rank=2 min=31798 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64380 max_load_reduction_pct=0.660470 moved_to_replicas=193815 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[28670, 32764, 98284, 10, 237, 32771, 23944, 45464] avg=32768.0000 max=98284 max_rank=2 min=10 min_rank=3 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32762, 33096, 33096, 33096, 32760, 32768, 31798] avg=32768.0000 max=33096 max_rank=2 min=31798 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64380 max_load_reduction_pct=0.660470 moved_to_replicas=193815 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[91, 9, 32656, 32285, 17641, 73349, 64083, 42030] avg=32768.0000 max=73349 max_rank=5 min=9 min_rank=1 imbalance_ratio=2.238434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[32768, 32762, 33096, 33096, 33096, 32760, 32768, 31798] avg=32768.0000 max=33096 max_rank=2 min=31798 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64380 max_load_reduction_pct=0.660470 moved_to_replicas=193815 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[31886, 32764, 33096, 32772, 32663, 32771, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=31886 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=65186 max_load_reduction_pct=0.663241 moved_to_replicas=155422 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[31886, 32764, 33096, 32772, 32663, 32771, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=31886 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=65186 max_load_reduction_pct=0.663241 moved_to_replicas=155422 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[28670, 32764, 98284, 10, 237, 32771, 23944, 45464] avg=32768.0000 max=98284 max_rank=2 min=10 min_rank=3 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[28670, 32764, 98284, 10, 237, 32771, 23944, 45464] avg=32768.0000 max=98284 max_rank=2 min=10 min_rank=3 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[28670, 32764, 98284, 10, 237, 32771, 23944, 45464] avg=32768.0000 max=98284 max_rank=2 min=10 min_rank=3 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[33094, 32763, 32656, 32285, 32058, 33098, 33096, 33094] avg=32768.0000 max=33098 max_rank=5 min=32058 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=40251 max_load_reduction_pct=0.548760 moved_to_replicas=164379 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[28670, 32764, 98284, 10, 237, 32771, 23944, 45464] avg=32768.0000 max=98284 max_rank=2 min=10 min_rank=3 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[31886, 32764, 33096, 32772, 32663, 32771, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=31886 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=65186 max_load_reduction_pct=0.663241 moved_to_replicas=155422 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[31886, 32764, 33096, 32772, 32663, 32771, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=31886 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=65186 max_load_reduction_pct=0.663241 moved_to_replicas=155422 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[28670, 32764, 98284, 10, 237, 32771, 23944, 45464] avg=32768.0000 max=98284 max_rank=2 min=10 min_rank=3 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[91, 9, 32656, 32285, 17641, 73349, 64083, 42030] avg=32768.0000 max=73349 max_rank=5 min=9 min_rank=1 imbalance_ratio=2.238434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[31886, 32764, 33096, 32772, 32663, 32771, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=31886 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=65186 max_load_reduction_pct=0.663241 moved_to_replicas=155422 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[91, 9, 32656, 32285, 17641, 73349, 64083, 42030] avg=32768.0000 max=73349 max_rank=5 min=9 min_rank=1 imbalance_ratio=2.238434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[31886, 32764, 33096, 32772, 32663, 32771, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=31886 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=65186 max_load_reduction_pct=0.663241 moved_to_replicas=155422 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[65440, 5, 11, 123, 65527, 32775, 65511, 32752] avg=32768.0000 max=65527 max_rank=4 min=5 min_rank=1 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[31886, 32764, 33096, 32772, 32663, 32771, 33094, 33098] avg=32768.0000 max=33098 max_rank=7 min=31886 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=65186 max_load_reduction_pct=0.663241 moved_to_replicas=155422 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[33094, 32763, 32656, 32285, 32058, 33098, 33096, 33094] avg=32768.0000 max=33098 max_rank=5 min=32058 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=40251 max_load_reduction_pct=0.548760 moved_to_replicas=164379 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[33094, 32763, 32656, 32285, 32058, 33098, 33096, 33094] avg=32768.0000 max=33098 max_rank=5 min=32058 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=40251 max_load_reduction_pct=0.548760 moved_to_replicas=164379 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[91, 9, 32656, 32285, 17641, 73349, 64083, 42030] avg=32768.0000 max=73349 max_rank=5 min=9 min_rank=1 imbalance_ratio=2.238434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[91, 9, 32656, 32285, 17641, 73349, 64083, 42030] avg=32768.0000 max=73349 max_rank=5 min=9 min_rank=1 imbalance_ratio=2.238434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[33096, 32436, 32426, 32467, 33096, 32775, 33096, 32752] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=194505 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[91, 9, 32656, 32285, 17641, 73349, 64083, 42030] avg=32768.0000 max=73349 max_rank=5 min=9 min_rank=1 imbalance_ratio=2.238434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[91, 9, 32656, 32285, 17641, 73349, 64083, 42030] avg=32768.0000 max=73349 max_rank=5 min=9 min_rank=1 imbalance_ratio=2.238434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[33094, 32763, 32656, 32285, 32058, 33098, 33096, 33094] avg=32768.0000 max=33098 max_rank=5 min=32058 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=40251 max_load_reduction_pct=0.548760 moved_to_replicas=164379 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[33094, 32763, 32656, 32285, 32058, 33098, 33096, 33094] avg=32768.0000 max=33098 max_rank=5 min=32058 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=40251 max_load_reduction_pct=0.548760 moved_to_replicas=164379 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[91, 9, 32656, 32285, 17641, 73349, 64083, 42030] avg=32768.0000 max=73349 max_rank=5 min=9 min_rank=1 imbalance_ratio=2.238434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[65440, 5, 11, 123, 65527, 32775, 65511, 32752] avg=32768.0000 max=65527 max_rank=4 min=5 min_rank=1 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[65440, 5, 11, 123, 65527, 32775, 65511, 32752] avg=32768.0000 max=65527 max_rank=4 min=5 min_rank=1 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[33094, 32763, 32656, 32285, 32058, 33098, 33096, 33094] avg=32768.0000 max=33098 max_rank=5 min=32058 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=40251 max_load_reduction_pct=0.548760 moved_to_replicas=164379 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[33094, 32763, 32656, 32285, 32058, 33098, 33096, 33094] avg=32768.0000 max=33098 max_rank=5 min=32058 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=40251 max_load_reduction_pct=0.548760 moved_to_replicas=164379 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[31657, 65446, 33879, 65529, 99, 32753, 19, 32762] avg=32768.0000 max=65529 max_rank=3 min=19 min_rank=6 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[33094, 32763, 32656, 32285, 32058, 33098, 33096, 33094] avg=32768.0000 max=33098 max_rank=5 min=32058 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=40251 max_load_reduction_pct=0.548760 moved_to_replicas=164379 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[33096, 32436, 32426, 32467, 33096, 32775, 33096, 32752] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=194505 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[33096, 32436, 32426, 32467, 33096, 32775, 33096, 32752] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=194505 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[65440, 5, 11, 123, 65527, 32775, 65511, 32752] avg=32768.0000 max=65527 max_rank=4 min=5 min_rank=1 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[65440, 5, 11, 123, 65527, 32775, 65511, 32752] avg=32768.0000 max=65527 max_rank=4 min=5 min_rank=1 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[32440, 33096, 33096, 33096, 32449, 32753, 32452, 32762] avg=32768.0000 max=33096 max_rank=1 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=130674 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[65440, 5, 11, 123, 65527, 32775, 65511, 32752] avg=32768.0000 max=65527 max_rank=4 min=5 min_rank=1 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[65440, 5, 11, 123, 65527, 32775, 65511, 32752] avg=32768.0000 max=65527 max_rank=4 min=5 min_rank=1 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[65440, 5, 11, 123, 65527, 32775, 65511, 32752] avg=32768.0000 max=65527 max_rank=4 min=5 min_rank=1 imbalance_ratio=1.999725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[33096, 32436, 32426, 32467, 33096, 32775, 33096, 32752] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=194505 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[33096, 32436, 32426, 32467, 33096, 32775, 33096, 32752] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=194505 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[31657, 65446, 33879, 65529, 99, 32753, 19, 32762] avg=32768.0000 max=65529 max_rank=3 min=19 min_rank=6 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[31657, 65446, 33879, 65529, 99, 32753, 19, 32762] avg=32768.0000 max=65529 max_rank=3 min=19 min_rank=6 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[33096, 32436, 32426, 32467, 33096, 32775, 33096, 32752] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=194505 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[33096, 32436, 32426, 32467, 33096, 32775, 33096, 32752] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=194505 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[38, 32812, 65522, 167, 32750, 65047, 33055, 32753] avg=32768.0000 max=65522 max_rank=2 min=38 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[33096, 32436, 32426, 32467, 33096, 32775, 33096, 32752] avg=32768.0000 max=33096 max_rank=0 min=32426 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32431 max_load_reduction_pct=0.494926 moved_to_replicas=194505 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[32440, 33096, 33096, 33096, 32449, 32753, 32452, 32762] avg=32768.0000 max=33096 max_rank=1 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=130674 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[32440, 33096, 33096, 33096, 32449, 32753, 32452, 32762] avg=32768.0000 max=33096 max_rank=1 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=130674 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[31657, 65446, 33879, 65529, 99, 32753, 19, 32762] avg=32768.0000 max=65529 max_rank=3 min=19 min_rank=6 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[31657, 65446, 33879, 65529, 99, 32753, 19, 32762] avg=32768.0000 max=65529 max_rank=3 min=19 min_rank=6 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[32464, 32812, 33096, 32118, 32750, 33096, 33055, 32753] avg=32768.0000 max=33096 max_rank=2 min=32118 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128526 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[31657, 65446, 33879, 65529, 99, 32753, 19, 32762] avg=32768.0000 max=65529 max_rank=3 min=19 min_rank=6 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[31657, 65446, 33879, 65529, 99, 32753, 19, 32762] avg=32768.0000 max=65529 max_rank=3 min=19 min_rank=6 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[31657, 65446, 33879, 65529, 99, 32753, 19, 32762] avg=32768.0000 max=65529 max_rank=3 min=19 min_rank=6 imbalance_ratio=1.999786 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[32440, 33096, 33096, 33096, 32449, 32753, 32452, 32762] avg=32768.0000 max=33096 max_rank=1 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=130674 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[32440, 33096, 33096, 33096, 32449, 32753, 32452, 32762] avg=32768.0000 max=33096 max_rank=1 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=130674 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[38, 32812, 65522, 167, 32750, 65047, 33055, 32753] avg=32768.0000 max=65522 max_rank=2 min=38 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[32440, 33096, 33096, 33096, 32449, 32753, 32452, 32762] avg=32768.0000 max=33096 max_rank=1 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=130674 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[38, 32812, 65522, 167, 32750, 65047, 33055, 32753] avg=32768.0000 max=65522 max_rank=2 min=38 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[32440, 33096, 33096, 33096, 32449, 32753, 32452, 32762] avg=32768.0000 max=33096 max_rank=1 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=130674 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[16, 77, 65571, 11, 98083, 32779, 32764, 32843] avg=32768.0000 max=98083 max_rank=4 min=11 min_rank=3 imbalance_ratio=2.993256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[32440, 33096, 33096, 33096, 32449, 32753, 32452, 32762] avg=32768.0000 max=33096 max_rank=1 min=32440 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32433 max_load_reduction_pct=0.494941 moved_to_replicas=130674 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[32464, 32812, 33096, 32118, 32750, 33096, 33055, 32753] avg=32768.0000 max=33096 max_rank=2 min=32118 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128526 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[32464, 32812, 33096, 32118, 32750, 33096, 33055, 32753] avg=32768.0000 max=33096 max_rank=2 min=32118 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128526 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[38, 32812, 65522, 167, 32750, 65047, 33055, 32753] avg=32768.0000 max=65522 max_rank=2 min=38 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[32248, 32552, 33096, 32766, 33096, 32779, 32764, 32843] avg=32768.0000 max=33096 max_rank=2 min=32248 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64987 max_load_reduction_pct=0.662571 moved_to_replicas=195050 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[38, 32812, 65522, 167, 32750, 65047, 33055, 32753] avg=32768.0000 max=65522 max_rank=2 min=38 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[38, 32812, 65522, 167, 32750, 65047, 33055, 32753] avg=32768.0000 max=65522 max_rank=2 min=38 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[38, 32812, 65522, 167, 32750, 65047, 33055, 32753] avg=32768.0000 max=65522 max_rank=2 min=38 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[38, 32812, 65522, 167, 32750, 65047, 33055, 32753] avg=32768.0000 max=65522 max_rank=2 min=38 min_rank=0 imbalance_ratio=1.999573 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[32464, 32812, 33096, 32118, 32750, 33096, 33055, 32753] avg=32768.0000 max=33096 max_rank=2 min=32118 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128526 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[16, 77, 65571, 11, 98083, 32779, 32764, 32843] avg=32768.0000 max=98083 max_rank=4 min=11 min_rank=3 imbalance_ratio=2.993256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[16, 77, 65571, 11, 98083, 32779, 32764, 32843] avg=32768.0000 max=98083 max_rank=4 min=11 min_rank=3 imbalance_ratio=2.993256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[32464, 32812, 33096, 32118, 32750, 33096, 33055, 32753] avg=32768.0000 max=33096 max_rank=2 min=32118 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128526 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[41, 32765, 32767, 32773, 64638, 32765, 32696, 33699] avg=32768.0000 max=64638 max_rank=4 min=41 min_rank=0 imbalance_ratio=1.972595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[32464, 32812, 33096, 32118, 32750, 33096, 33055, 32753] avg=32768.0000 max=33096 max_rank=2 min=32118 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128526 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[32464, 32812, 33096, 32118, 32750, 33096, 33055, 32753] avg=32768.0000 max=33096 max_rank=2 min=32118 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128526 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[32464, 32812, 33096, 32118, 32750, 33096, 33055, 32753] avg=32768.0000 max=33096 max_rank=2 min=32118 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32426 max_load_reduction_pct=0.494887 moved_to_replicas=128526 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[32248, 32552, 33096, 32766, 33096, 32779, 32764, 32843] avg=32768.0000 max=33096 max_rank=2 min=32248 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64987 max_load_reduction_pct=0.662571 moved_to_replicas=195050 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[32248, 32552, 33096, 32766, 33096, 32779, 32764, 32843] avg=32768.0000 max=33096 max_rank=2 min=32248 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64987 max_load_reduction_pct=0.662571 moved_to_replicas=195050 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[32186, 32765, 32767, 32773, 33096, 32765, 32696, 33096] avg=32768.0000 max=33096 max_rank=4 min=32186 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31542 max_load_reduction_pct=0.487979 moved_to_replicas=64273 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[16, 77, 65571, 11, 98083, 32779, 32764, 32843] avg=32768.0000 max=98083 max_rank=4 min=11 min_rank=3 imbalance_ratio=2.993256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[16, 77, 65571, 11, 98083, 32779, 32764, 32843] avg=32768.0000 max=98083 max_rank=4 min=11 min_rank=3 imbalance_ratio=2.993256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[16, 77, 65571, 11, 98083, 32779, 32764, 32843] avg=32768.0000 max=98083 max_rank=4 min=11 min_rank=3 imbalance_ratio=2.993256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[16, 77, 65571, 11, 98083, 32779, 32764, 32843] avg=32768.0000 max=98083 max_rank=4 min=11 min_rank=3 imbalance_ratio=2.993256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[16, 77, 65571, 11, 98083, 32779, 32764, 32843] avg=32768.0000 max=98083 max_rank=4 min=11 min_rank=3 imbalance_ratio=2.993256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[32248, 32552, 33096, 32766, 33096, 32779, 32764, 32843] avg=32768.0000 max=33096 max_rank=2 min=32248 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64987 max_load_reduction_pct=0.662571 moved_to_replicas=195050 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[41, 32765, 32767, 32773, 64638, 32765, 32696, 33699] avg=32768.0000 max=64638 max_rank=4 min=41 min_rank=0 imbalance_ratio=1.972595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[97923, 540, 65499, 32775, 17, 32621, 32768, 1] avg=32768.0000 max=97923 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.988373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[32248, 32552, 33096, 32766, 33096, 32779, 32764, 32843] avg=32768.0000 max=33096 max_rank=2 min=32248 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64987 max_load_reduction_pct=0.662571 moved_to_replicas=195050 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[41, 32765, 32767, 32773, 64638, 32765, 32696, 33699] avg=32768.0000 max=64638 max_rank=4 min=41 min_rank=0 imbalance_ratio=1.972595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[32248, 32552, 33096, 32766, 33096, 32779, 32764, 32843] avg=32768.0000 max=33096 max_rank=2 min=32248 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64987 max_load_reduction_pct=0.662571 moved_to_replicas=195050 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[32248, 32552, 33096, 32766, 33096, 32779, 32764, 32843] avg=32768.0000 max=33096 max_rank=2 min=32248 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64987 max_load_reduction_pct=0.662571 moved_to_replicas=195050 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[32248, 32552, 33096, 32766, 33096, 32779, 32764, 32843] avg=32768.0000 max=33096 max_rank=2 min=32248 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=64987 max_load_reduction_pct=0.662571 moved_to_replicas=195050 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[32186, 32765, 32767, 32773, 33096, 32765, 32696, 33096] avg=32768.0000 max=33096 max_rank=4 min=32186 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31542 max_load_reduction_pct=0.487979 moved_to_replicas=64273 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[33096, 32943, 33096, 32775, 32117, 32621, 32768, 32728] avg=32768.0000 max=33096 max_rank=0 min=32117 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64827 max_load_reduction_pct=0.662020 moved_to_replicas=194626 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[32186, 32765, 32767, 32773, 33096, 32765, 32696, 33096] avg=32768.0000 max=33096 max_rank=4 min=32186 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31542 max_load_reduction_pct=0.487979 moved_to_replicas=64273 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[41, 32765, 32767, 32773, 64638, 32765, 32696, 33699] avg=32768.0000 max=64638 max_rank=4 min=41 min_rank=0 imbalance_ratio=1.972595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[41, 32765, 32767, 32773, 64638, 32765, 32696, 33699] avg=32768.0000 max=64638 max_rank=4 min=41 min_rank=0 imbalance_ratio=1.972595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[41, 32765, 32767, 32773, 64638, 32765, 32696, 33699] avg=32768.0000 max=64638 max_rank=4 min=41 min_rank=0 imbalance_ratio=1.972595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[41, 32765, 32767, 32773, 64638, 32765, 32696, 33699] avg=32768.0000 max=64638 max_rank=4 min=41 min_rank=0 imbalance_ratio=1.972595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[41, 32765, 32767, 32773, 64638, 32765, 32696, 33699] avg=32768.0000 max=64638 max_rank=4 min=41 min_rank=0 imbalance_ratio=1.972595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[32186, 32765, 32767, 32773, 33096, 32765, 32696, 33096] avg=32768.0000 max=33096 max_rank=4 min=32186 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31542 max_load_reduction_pct=0.487979 moved_to_replicas=64273 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[97923, 540, 65499, 32775, 17, 32621, 32768, 1] avg=32768.0000 max=97923 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.988373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[32186, 32765, 32767, 32773, 33096, 32765, 32696, 33096] avg=32768.0000 max=33096 max_rank=4 min=32186 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31542 max_load_reduction_pct=0.487979 moved_to_replicas=64273 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[97923, 540, 65499, 32775, 17, 32621, 32768, 1] avg=32768.0000 max=97923 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.988373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[32186, 32765, 32767, 32773, 33096, 32765, 32696, 33096] avg=32768.0000 max=33096 max_rank=4 min=32186 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31542 max_load_reduction_pct=0.487979 moved_to_replicas=64273 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[32186, 32765, 32767, 32773, 33096, 32765, 32696, 33096] avg=32768.0000 max=33096 max_rank=4 min=32186 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31542 max_load_reduction_pct=0.487979 moved_to_replicas=64273 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[32186, 32765, 32767, 32773, 33096, 32765, 32696, 33096] avg=32768.0000 max=33096 max_rank=4 min=32186 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=31542 max_load_reduction_pct=0.487979 moved_to_replicas=64273 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[33096, 32943, 33096, 32775, 32117, 32621, 32768, 32728] avg=32768.0000 max=33096 max_rank=0 min=32117 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64827 max_load_reduction_pct=0.662020 moved_to_replicas=194626 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[97923, 540, 65499, 32775, 17, 32621, 32768, 1] avg=32768.0000 max=97923 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.988373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[33096, 32943, 33096, 32775, 32117, 32621, 32768, 32728] avg=32768.0000 max=33096 max_rank=0 min=32117 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64827 max_load_reduction_pct=0.662020 moved_to_replicas=194626 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[97923, 540, 65499, 32775, 17, 32621, 32768, 1] avg=32768.0000 max=97923 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.988373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[97923, 540, 65499, 32775, 17, 32621, 32768, 1] avg=32768.0000 max=97923 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.988373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[97923, 540, 65499, 32775, 17, 32621, 32768, 1] avg=32768.0000 max=97923 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.988373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[97923, 540, 65499, 32775, 17, 32621, 32768, 1] avg=32768.0000 max=97923 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.988373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[33096, 32943, 33096, 32775, 32117, 32621, 32768, 32728] avg=32768.0000 max=33096 max_rank=0 min=32117 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64827 max_load_reduction_pct=0.662020 moved_to_replicas=194626 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[33096, 32943, 33096, 32775, 32117, 32621, 32768, 32728] avg=32768.0000 max=33096 max_rank=0 min=32117 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64827 max_load_reduction_pct=0.662020 moved_to_replicas=194626 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[33096, 32943, 33096, 32775, 32117, 32621, 32768, 32728] avg=32768.0000 max=33096 max_rank=0 min=32117 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64827 max_load_reduction_pct=0.662020 moved_to_replicas=194626 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[33096, 32943, 33096, 32775, 32117, 32621, 32768, 32728] avg=32768.0000 max=33096 max_rank=0 min=32117 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64827 max_load_reduction_pct=0.662020 moved_to_replicas=194626 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[33096, 32943, 33096, 32775, 32117, 32621, 32768, 32728] avg=32768.0000 max=33096 max_rank=0 min=32117 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=64827 max_load_reduction_pct=0.662020 moved_to_replicas=194626 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=32 total_expected_recv=1048914 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254336 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=32 total_expected_recv=1047052 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253696 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=32 total_expected_recv=1050401 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=250112 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=32 total_expected_recv=1052094 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=248576 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=32 total_expected_recv=1046743 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=255104 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=32 total_expected_recv=1045249 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254080 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=32 total_expected_recv=1050746 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=251392 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=32 total_expected_recv=1047409 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254208 peak_demand=33097 required_capacity=1.010040 + [2026-08-30 05:10:30] iteration 9/ 12 | consumed samples: 144 | elapsed time per iteration (ms): 560.3 | throughput per GPU (TFLOP/s/GPU): 595.0 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.274881E+00 | loss scale: 1.0 | grad norm: 5.713 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=40960 expected_recv=64836 expected_overflow=0 exact_recv_per_rank=[31358, 33096, 33096, 32210, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31358, 33096, 33096, 32210, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31358, 33096, 33096, 32210, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=40960 expected_recv=63186 expected_overflow=0 exact_recv_per_rank=[31358, 33096, 33096, 32210, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31358, 33096, 33096, 32210, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[31358, 33096, 33096, 32210, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=2 budget=40960 expected_recv=64836 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31358, 33096, 33096, 32210, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=2 budget=40960 expected_recv=63186 actual_valid_recv=63872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=40960 expected_recv=65306 expected_overflow=0 exact_recv_per_rank=[31358, 33096, 33096, 32210, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=63872 slack_slots=18048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=2 budget=40960 expected_recv=65306 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32137, 32756, 33096, 33096, 32769, 32772, 32776, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65484 expected_overflow=0 exact_recv_per_rank=[32137, 32756, 33096, 33096, 32769, 32772, 32776, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[32137, 32756, 33096, 33096, 32769, 32772, 32776, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65484 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65533 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=40960 expected_recv=64323 expected_overflow=0 exact_recv_per_rank=[32137, 32756, 33096, 33096, 32769, 32772, 32776, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65489 expected_overflow=0 exact_recv_per_rank=[32137, 32756, 33096, 33096, 32769, 32772, 32776, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[32137, 32756, 33096, 33096, 32769, 32772, 32776, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=2 budget=40960 expected_recv=64323 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65489 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[32137, 32756, 33096, 33096, 32769, 32772, 32776, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65540 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32137, 32756, 33096, 33096, 32769, 32772, 32776, 32742] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=40960 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[32790, 32678, 33094, 32773, 32434, 33096, 32503, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=2 budget=40960 expected_recv=65535 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32790, 32678, 33094, 32773, 32434, 33096, 32503, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65398 expected_overflow=0 exact_recv_per_rank=[32790, 32678, 33094, 32773, 32434, 33096, 32503, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=2 budget=40960 expected_recv=66190 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65398 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65584 expected_overflow=0 exact_recv_per_rank=[32790, 32678, 33094, 32773, 32434, 33096, 32503, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[32790, 32678, 33094, 32773, 32434, 33096, 32503, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=40960 expected_recv=64982 expected_overflow=0 exact_recv_per_rank=[32790, 32678, 33094, 32773, 32434, 33096, 32503, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65584 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=40960 expected_recv=64859 expected_overflow=0 exact_recv_per_rank=[32790, 32678, 33094, 32773, 32434, 33096, 32503, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65543 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=2 budget=40960 expected_recv=64982 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31703, 32862, 33096, 32422, 33096, 33095, 33097, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65540 expected_overflow=0 exact_recv_per_rank=[32790, 32678, 33094, 32773, 32434, 33096, 32503, 32776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=40960 expected_recv=65959 expected_overflow=0 exact_recv_per_rank=[31703, 32862, 33096, 32422, 33096, 33095, 33097, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=2 budget=40960 expected_recv=64859 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[31703, 32862, 33096, 32422, 33096, 33095, 33097, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=2 budget=40960 expected_recv=65540 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=2 budget=40960 expected_recv=65959 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=40960 expected_recv=63173 expected_overflow=0 exact_recv_per_rank=[31703, 32862, 33096, 32422, 33096, 33095, 33097, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=40960 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[31703, 32862, 33096, 32422, 33096, 33095, 33097, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=2 budget=40960 expected_recv=63173 actual_valid_recv=64000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31703, 32862, 33096, 32422, 33096, 33095, 33097, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=2 budget=40960 expected_recv=65545 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=64000 slack_slots=17920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31703, 32862, 33096, 32422, 33096, 33095, 33097, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32387, 33096, 33096, 32783, 32766, 33096, 32771, 32149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32387, 33096, 33096, 32783, 32766, 33096, 32771, 32149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=40960 expected_recv=64844 expected_overflow=0 exact_recv_per_rank=[31703, 32862, 33096, 32422, 33096, 33095, 33097, 32773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32387, 33096, 33096, 32783, 32766, 33096, 32771, 32149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=2 budget=40960 expected_recv=64844 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=40960 expected_recv=64794 expected_overflow=0 exact_recv_per_rank=[32387, 33096, 33096, 32783, 32766, 33096, 32771, 32149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=40960 expected_recv=64288 expected_overflow=0 exact_recv_per_rank=[32387, 33096, 33096, 32783, 32766, 33096, 32771, 32149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=2 budget=40960 expected_recv=64794 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[32387, 33096, 33096, 32783, 32766, 33096, 32771, 32149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=2 budget=40960 expected_recv=64288 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65544 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[32387, 33096, 33096, 32783, 32766, 33096, 32771, 32149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32733, 32310, 32779, 33096, 31938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65621 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32733, 32310, 32779, 33096, 31938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65553 expected_overflow=0 exact_recv_per_rank=[32387, 33096, 33096, 32783, 32766, 33096, 32771, 32149] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65533 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32733, 32310, 32779, 33096, 31938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65621 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=2 budget=40960 expected_recv=65553 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32733, 32310, 32779, 33096, 31938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65546 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=40960 expected_recv=63766 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32733, 32310, 32779, 33096, 31938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32733, 32310, 32779, 33096, 31938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=2 budget=40960 expected_recv=63766 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65404 expected_overflow=0 exact_recv_per_rank=[32804, 32691, 32771, 32952, 33096, 32765, 32378, 32687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[32804, 32691, 32771, 32952, 33096, 32765, 32378, 32687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65406 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32733, 32310, 32779, 33096, 31938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65373 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32733, 32310, 32779, 33096, 31938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65404 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65529 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65533 expected_overflow=0 exact_recv_per_rank=[32804, 32691, 32771, 32952, 33096, 32765, 32378, 32687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65406 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=2 budget=40960 expected_recv=65373 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65598 expected_overflow=0 exact_recv_per_rank=[32804, 32691, 32771, 32952, 33096, 32765, 32378, 32687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65338 expected_overflow=0 exact_recv_per_rank=[32804, 32691, 32771, 32952, 33096, 32765, 32378, 32687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65533 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65598 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=40960 expected_recv=64785 expected_overflow=0 exact_recv_per_rank=[32804, 32691, 32771, 32952, 33096, 32765, 32378, 32687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65338 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=2 budget=40960 expected_recv=64785 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33042, 31886, 33096, 32615, 33096, 32725, 32921, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=40960 expected_recv=63690 expected_overflow=0 exact_recv_per_rank=[33042, 31886, 33096, 32615, 33096, 32725, 32921, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=2 budget=40960 expected_recv=63690 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32804, 32691, 32771, 32952, 33096, 32765, 32378, 32687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65909 expected_overflow=0 exact_recv_per_rank=[32804, 32691, 32771, 32952, 33096, 32765, 32378, 32687] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65477 expected_overflow=0 exact_recv_per_rank=[33042, 31886, 33096, 32615, 33096, 32725, 32921, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=2 budget=40960 expected_recv=65909 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65997 expected_overflow=0 exact_recv_per_rank=[33042, 31886, 33096, 32615, 33096, 32725, 32921, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[33042, 31886, 33096, 32615, 33096, 32725, 32921, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65477 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=40960 expected_recv=66017 expected_overflow=0 exact_recv_per_rank=[33042, 31886, 33096, 32615, 33096, 32725, 32921, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65997 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65532 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=2 budget=40960 expected_recv=66017 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32772, 33096, 32774, 32011, 32773, 32526, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[33096, 32772, 33096, 32774, 32011, 32773, 32526, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65537 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33042, 31886, 33096, 32615, 33096, 32725, 32921, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65191 expected_overflow=0 exact_recv_per_rank=[33042, 31886, 33096, 32615, 33096, 32725, 32921, 32763] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[33096, 32772, 33096, 32774, 32011, 32773, 32526, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32772, 33096, 32774, 32011, 32773, 32526, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32772, 33096, 32774, 32011, 32773, 32526, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65000 expected_overflow=0 exact_recv_per_rank=[33096, 32772, 33096, 32774, 32011, 32773, 32526, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=2 budget=40960 expected_recv=65191 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65544 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65000 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=40960 expected_recv=64722 expected_overflow=0 exact_recv_per_rank=[32272, 32764, 31627, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=40960 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[32272, 32764, 31627, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=2 budget=40960 expected_recv=64722 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=2 budget=40960 expected_recv=65532 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=40960 expected_recv=64085 expected_overflow=0 exact_recv_per_rank=[33096, 32772, 33096, 32774, 32011, 32773, 32526, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[33096, 32772, 33096, 32774, 32011, 32773, 32526, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32272, 32764, 31627, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=40960 expected_recv=64595 expected_overflow=0 exact_recv_per_rank=[32272, 32764, 31627, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32272, 32764, 31627, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32272, 32764, 31627, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=2 budget=40960 expected_recv=64085 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=2 budget=40960 expected_recv=65546 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=2 budget=40960 expected_recv=64595 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=40960 expected_recv=64891 expected_overflow=0 exact_recv_per_rank=[33096, 32440, 32447, 32694, 33096, 32518, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=40960 expected_recv=64884 expected_overflow=0 exact_recv_per_rank=[33096, 32440, 32447, 32694, 33096, 32518, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=2 budget=40960 expected_recv=64891 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=2 budget=40960 expected_recv=64884 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32272, 32764, 31627, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65081 expected_overflow=0 exact_recv_per_rank=[33096, 32440, 32447, 32694, 33096, 32518, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=40960 expected_recv=64669 expected_overflow=0 exact_recv_per_rank=[32272, 32764, 31627, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32440, 32447, 32694, 33096, 32518, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32440, 32447, 32694, 33096, 32518, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65513 expected_overflow=0 exact_recv_per_rank=[33096, 32440, 32447, 32694, 33096, 32518, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65081 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=2 budget=40960 expected_recv=64669 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65513 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32601, 33096, 33096, 33096, 33096, 32460, 32420, 32279] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32601, 33096, 33096, 33096, 33096, 32460, 32420, 32279] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=40960 expected_recv=64924 expected_overflow=0 exact_recv_per_rank=[32601, 33096, 33096, 33096, 33096, 32460, 32420, 32279] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32440, 32447, 32694, 33096, 32518, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65343 expected_overflow=0 exact_recv_per_rank=[33096, 32440, 32447, 32694, 33096, 32518, 33096, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=40960 expected_recv=65240 expected_overflow=0 exact_recv_per_rank=[32601, 33096, 33096, 33096, 33096, 32460, 32420, 32279] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=40960 expected_recv=64852 expected_overflow=0 exact_recv_per_rank=[32601, 33096, 33096, 33096, 33096, 32460, 32420, 32279] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=2 budget=40960 expected_recv=64924 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=40960 expected_recv=64504 expected_overflow=0 exact_recv_per_rank=[32601, 33096, 33096, 33096, 33096, 32460, 32420, 32279] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=2 budget=40960 expected_recv=65343 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=2 budget=40960 expected_recv=65240 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=2 budget=40960 expected_recv=64852 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32478, 32800, 33096, 32742, 32756, 32507, 33011, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=2 budget=40960 expected_recv=64504 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65587 expected_overflow=0 exact_recv_per_rank=[32478, 32800, 33096, 32742, 32756, 32507, 33011, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65587 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65104 expected_overflow=0 exact_recv_per_rank=[32478, 32800, 33096, 32742, 32756, 32507, 33011, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32601, 33096, 33096, 33096, 33096, 32460, 32420, 32279] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32601, 33096, 33096, 33096, 33096, 32460, 32420, 32279] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65104 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=40960 expected_recv=64937 expected_overflow=0 exact_recv_per_rank=[32478, 32800, 33096, 32742, 32756, 32507, 33011, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65949 expected_overflow=0 exact_recv_per_rank=[32478, 32800, 33096, 32742, 32756, 32507, 33011, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65506 expected_overflow=0 exact_recv_per_rank=[32478, 32800, 33096, 32742, 32756, 32507, 33011, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[31950, 32907, 33096, 32764, 33094, 32783, 32765, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=2 budget=40960 expected_recv=64937 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65949 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65506 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65765 expected_overflow=0 exact_recv_per_rank=[31950, 32907, 33096, 32764, 33094, 32783, 32765, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65765 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65554 expected_overflow=0 exact_recv_per_rank=[31950, 32907, 33096, 32764, 33094, 32783, 32765, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65496 expected_overflow=0 exact_recv_per_rank=[32478, 32800, 33096, 32742, 32756, 32507, 33011, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65517 expected_overflow=0 exact_recv_per_rank=[32478, 32800, 33096, 32742, 32756, 32507, 33011, 32754] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65554 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65496 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=2 budget=40960 expected_recv=65517 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=40960 expected_recv=63962 expected_overflow=0 exact_recv_per_rank=[31950, 32907, 33096, 32764, 33094, 32783, 32765, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[31950, 32907, 33096, 32764, 33094, 32783, 32765, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=40960 expected_recv=64146 expected_overflow=0 exact_recv_per_rank=[32763, 33049, 32004, 32770, 33096, 32777, 32589, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65566 expected_overflow=0 exact_recv_per_rank=[31950, 32907, 33096, 32764, 33094, 32783, 32765, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65531 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=2 budget=40960 expected_recv=63962 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=2 budget=40960 expected_recv=64146 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=40960 expected_recv=66072 expected_overflow=0 exact_recv_per_rank=[32763, 33049, 32004, 32770, 33096, 32777, 32589, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65566 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=2 budget=40960 expected_recv=66072 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65548 expected_overflow=0 exact_recv_per_rank=[32763, 33049, 32004, 32770, 33096, 32777, 32589, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65548 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65530 expected_overflow=0 exact_recv_per_rank=[31950, 32907, 33096, 32764, 33094, 32783, 32765, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=40960 expected_recv=66189 expected_overflow=0 exact_recv_per_rank=[31950, 32907, 33096, 32764, 33094, 32783, 32765, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=13 calls=2 budget=40960 expected_recv=65530 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65233 expected_overflow=0 exact_recv_per_rank=[32763, 33049, 32004, 32770, 33096, 32777, 32589, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=13 calls=2 budget=40960 expected_recv=66189 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65366 expected_overflow=0 exact_recv_per_rank=[32763, 33049, 32004, 32770, 33096, 32777, 32589, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32206, 33096, 32765, 32423, 33096, 32768, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32763, 33049, 32004, 32770, 33096, 32777, 32589, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65233 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=40960 expected_recv=64227 expected_overflow=0 exact_recv_per_rank=[33096, 32206, 33096, 32765, 32423, 33096, 32768, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65366 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=15 calls=2 budget=40960 expected_recv=64227 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32206, 33096, 32765, 32423, 33096, 32768, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[32763, 33049, 32004, 32770, 33096, 32777, 32589, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32763, 33049, 32004, 32770, 33096, 32777, 32589, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[33096, 32206, 33096, 32765, 32423, 33096, 32768, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=14 calls=2 budget=40960 expected_recv=65539 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65416 expected_overflow=0 exact_recv_per_rank=[33096, 32206, 33096, 32765, 32423, 33096, 32768, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32206, 33096, 32765, 32423, 33096, 32768, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65536 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[31358, 26843, 16717, 25747, 7327, 27053, 49454, 77645] avg=32768.0000 max=77645 max_rank=7 min=7327 min_rank=4 imbalance_ratio=2.369537 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65416 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[31358, 26843, 16717, 25747, 7327, 27053, 49454, 77645] avg=32768.0000 max=77645 max_rank=7 min=7327 min_rank=4 imbalance_ratio=2.369537 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[31358, 26843, 16717, 25747, 7327, 27053, 49454, 77645] avg=32768.0000 max=77645 max_rank=7 min=7327 min_rank=4 imbalance_ratio=2.369537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[31358, 33096, 33096, 32210, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=44549 max_load_reduction_pct=0.573752 moved_to_replicas=120023 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[31358, 33096, 33096, 32210, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=44549 max_load_reduction_pct=0.573752 moved_to_replicas=120023 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[33096, 32206, 33096, 32765, 32423, 33096, 32768, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65002 expected_overflow=0 exact_recv_per_rank=[33096, 32206, 33096, 32765, 32423, 33096, 32768, 32694] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[31358, 33096, 33096, 32210, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=44549 max_load_reduction_pct=0.573752 moved_to_replicas=120023 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65531 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=2 budget=40960 expected_recv=65002 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[31358, 26843, 16717, 25747, 7327, 27053, 49454, 77645] avg=32768.0000 max=77645 max_rank=7 min=7327 min_rank=4 imbalance_ratio=2.369537 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[31358, 26843, 16717, 25747, 7327, 27053, 49454, 77645] avg=32768.0000 max=77645 max_rank=7 min=7327 min_rank=4 imbalance_ratio=2.369537 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[31358, 26843, 16717, 25747, 7327, 27053, 49454, 77645] avg=32768.0000 max=77645 max_rank=7 min=7327 min_rank=4 imbalance_ratio=2.369537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[3186, 32756, 99377, 61283, 32769, 11, 20, 32742] avg=32768.0000 max=99377 max_rank=2 min=11 min_rank=5 imbalance_ratio=3.032745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[3186, 32756, 99377, 61283, 32769, 11, 20, 32742] avg=32768.0000 max=99377 max_rank=2 min=11 min_rank=5 imbalance_ratio=3.032745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[31358, 33096, 33096, 32210, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=44549 max_load_reduction_pct=0.573752 moved_to_replicas=120023 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[3186, 32756, 99377, 61283, 32769, 11, 20, 32742] avg=32768.0000 max=99377 max_rank=2 min=11 min_rank=5 imbalance_ratio=3.032745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[31358, 33096, 33096, 32210, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=44549 max_load_reduction_pct=0.573752 moved_to_replicas=120023 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[31358, 33096, 33096, 32210, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=44549 max_load_reduction_pct=0.573752 moved_to_replicas=120023 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[31358, 26843, 16717, 25747, 7327, 27053, 49454, 77645] avg=32768.0000 max=77645 max_rank=7 min=7327 min_rank=4 imbalance_ratio=2.369537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[32137, 32756, 33096, 33096, 32769, 32772, 32776, 32742] avg=32768.0000 max=33096 max_rank=2 min=32137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=66281 max_load_reduction_pct=0.666965 moved_to_replicas=190086 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[31358, 26843, 16717, 25747, 7327, 27053, 49454, 77645] avg=32768.0000 max=77645 max_rank=7 min=7327 min_rank=4 imbalance_ratio=2.369537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[32137, 32756, 33096, 33096, 32769, 32772, 32776, 32742] avg=32768.0000 max=33096 max_rank=2 min=32137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=66281 max_load_reduction_pct=0.666965 moved_to_replicas=190086 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[32137, 32756, 33096, 33096, 32769, 32772, 32776, 32742] avg=32768.0000 max=33096 max_rank=2 min=32137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=66281 max_load_reduction_pct=0.666965 moved_to_replicas=190086 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[31358, 33096, 33096, 32210, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=44549 max_load_reduction_pct=0.573752 moved_to_replicas=120023 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[3186, 32756, 99377, 61283, 32769, 11, 20, 32742] avg=32768.0000 max=99377 max_rank=2 min=11 min_rank=5 imbalance_ratio=3.032745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[31358, 33096, 33096, 32210, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31358 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=44549 max_load_reduction_pct=0.573752 moved_to_replicas=120023 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[3186, 32756, 99377, 61283, 32769, 11, 20, 32742] avg=32768.0000 max=99377 max_rank=2 min=11 min_rank=5 imbalance_ratio=3.032745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[3186, 32756, 99377, 61283, 32769, 11, 20, 32742] avg=32768.0000 max=99377 max_rank=2 min=11 min_rank=5 imbalance_ratio=3.032745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[32790, 32678, 54041, 32773, 16, 65514, 11556, 32776] avg=32768.0000 max=65514 max_rank=5 min=16 min_rank=4 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[32790, 32678, 54041, 32773, 16, 65514, 11556, 32776] avg=32768.0000 max=65514 max_rank=5 min=16 min_rank=4 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[32790, 32678, 54041, 32773, 16, 65514, 11556, 32776] avg=32768.0000 max=65514 max_rank=5 min=16 min_rank=4 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[32137, 32756, 33096, 33096, 32769, 32772, 32776, 32742] avg=32768.0000 max=33096 max_rank=2 min=32137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=66281 max_load_reduction_pct=0.666965 moved_to_replicas=190086 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[32137, 32756, 33096, 33096, 32769, 32772, 32776, 32742] avg=32768.0000 max=33096 max_rank=2 min=32137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=66281 max_load_reduction_pct=0.666965 moved_to_replicas=190086 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[32137, 32756, 33096, 33096, 32769, 32772, 32776, 32742] avg=32768.0000 max=33096 max_rank=2 min=32137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=66281 max_load_reduction_pct=0.666965 moved_to_replicas=190086 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[32790, 32678, 33094, 32773, 32434, 33096, 32503, 32776] avg=32768.0000 max=33096 max_rank=5 min=32434 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=104639 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[32790, 32678, 33094, 32773, 32434, 33096, 32503, 32776] avg=32768.0000 max=33096 max_rank=5 min=32434 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=104639 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[3186, 32756, 99377, 61283, 32769, 11, 20, 32742] avg=32768.0000 max=99377 max_rank=2 min=11 min_rank=5 imbalance_ratio=3.032745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[3186, 32756, 99377, 61283, 32769, 11, 20, 32742] avg=32768.0000 max=99377 max_rank=2 min=11 min_rank=5 imbalance_ratio=3.032745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[32790, 32678, 33094, 32773, 32434, 33096, 32503, 32776] avg=32768.0000 max=33096 max_rank=5 min=32434 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=104639 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[32137, 32756, 33096, 33096, 32769, 32772, 32776, 32742] avg=32768.0000 max=33096 max_rank=2 min=32137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=66281 max_load_reduction_pct=0.666965 moved_to_replicas=190086 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[32790, 32678, 54041, 32773, 16, 65514, 11556, 32776] avg=32768.0000 max=65514 max_rank=5 min=16 min_rank=4 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[32137, 32756, 33096, 33096, 32769, 32772, 32776, 32742] avg=32768.0000 max=33096 max_rank=2 min=32137 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=66281 max_load_reduction_pct=0.666965 moved_to_replicas=190086 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[32790, 32678, 54041, 32773, 16, 65514, 11556, 32776] avg=32768.0000 max=65514 max_rank=5 min=16 min_rank=4 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[32790, 32678, 54041, 32773, 16, 65514, 11556, 32776] avg=32768.0000 max=65514 max_rank=5 min=16 min_rank=4 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[16, 28561, 64783, 11, 65507, 15116, 55377, 32773] avg=32768.0000 max=65507 max_rank=4 min=11 min_rank=3 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[16, 28561, 64783, 11, 65507, 15116, 55377, 32773] avg=32768.0000 max=65507 max_rank=4 min=11 min_rank=3 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[16, 28561, 64783, 11, 65507, 15116, 55377, 32773] avg=32768.0000 max=65507 max_rank=4 min=11 min_rank=3 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[32790, 32678, 33094, 32773, 32434, 33096, 32503, 32776] avg=32768.0000 max=33096 max_rank=5 min=32434 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=104639 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[32790, 32678, 33094, 32773, 32434, 33096, 32503, 32776] avg=32768.0000 max=33096 max_rank=5 min=32434 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=104639 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[32790, 32678, 33094, 32773, 32434, 33096, 32503, 32776] avg=32768.0000 max=33096 max_rank=5 min=32434 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=104639 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[31703, 32862, 33096, 32422, 33096, 33095, 33097, 32773] avg=32768.0000 max=33097 max_rank=6 min=31703 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32410 max_load_reduction_pct=0.494756 moved_to_replicas=174993 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[31703, 32862, 33096, 32422, 33096, 33095, 33097, 32773] avg=32768.0000 max=33097 max_rank=6 min=31703 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32410 max_load_reduction_pct=0.494756 moved_to_replicas=174993 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[32790, 32678, 54041, 32773, 16, 65514, 11556, 32776] avg=32768.0000 max=65514 max_rank=5 min=16 min_rank=4 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[32790, 32678, 54041, 32773, 16, 65514, 11556, 32776] avg=32768.0000 max=65514 max_rank=5 min=16 min_rank=4 imbalance_ratio=1.999329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[31703, 32862, 33096, 32422, 33096, 33095, 33097, 32773] avg=32768.0000 max=33097 max_rank=6 min=31703 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32410 max_load_reduction_pct=0.494756 moved_to_replicas=174993 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[32790, 32678, 33094, 32773, 32434, 33096, 32503, 32776] avg=32768.0000 max=33096 max_rank=5 min=32434 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=104639 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[16, 28561, 64783, 11, 65507, 15116, 55377, 32773] avg=32768.0000 max=65507 max_rank=4 min=11 min_rank=3 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[16, 28561, 64783, 11, 65507, 15116, 55377, 32773] avg=32768.0000 max=65507 max_rank=4 min=11 min_rank=3 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[32790, 32678, 33094, 32773, 32434, 33096, 32503, 32776] avg=32768.0000 max=33096 max_rank=5 min=32434 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32418 max_load_reduction_pct=0.494826 moved_to_replicas=104639 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[16, 28561, 64783, 11, 65507, 15116, 55377, 32773] avg=32768.0000 max=65507 max_rank=4 min=11 min_rank=3 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[25, 98219, 34321, 32783, 5, 33563, 32771, 30457] avg=32768.0000 max=98219 max_rank=1 min=5 min_rank=4 imbalance_ratio=2.997406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[25, 98219, 34321, 32783, 5, 33563, 32771, 30457] avg=32768.0000 max=98219 max_rank=1 min=5 min_rank=4 imbalance_ratio=2.997406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[25, 98219, 34321, 32783, 5, 33563, 32771, 30457] avg=32768.0000 max=98219 max_rank=1 min=5 min_rank=4 imbalance_ratio=2.997406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[31703, 32862, 33096, 32422, 33096, 33095, 33097, 32773] avg=32768.0000 max=33097 max_rank=6 min=31703 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32410 max_load_reduction_pct=0.494756 moved_to_replicas=174993 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[31703, 32862, 33096, 32422, 33096, 33095, 33097, 32773] avg=32768.0000 max=33097 max_rank=6 min=31703 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32410 max_load_reduction_pct=0.494756 moved_to_replicas=174993 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[31703, 32862, 33096, 32422, 33096, 33095, 33097, 32773] avg=32768.0000 max=33097 max_rank=6 min=31703 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32410 max_load_reduction_pct=0.494756 moved_to_replicas=174993 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[32387, 33096, 33096, 32783, 32766, 33096, 32771, 32149] avg=32768.0000 max=33096 max_rank=1 min=32149 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.663039 moved_to_replicas=133636 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[32387, 33096, 33096, 32783, 32766, 33096, 32771, 32149] avg=32768.0000 max=33096 max_rank=1 min=32149 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.663039 moved_to_replicas=133636 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[16, 28561, 64783, 11, 65507, 15116, 55377, 32773] avg=32768.0000 max=65507 max_rank=4 min=11 min_rank=3 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[32387, 33096, 33096, 32783, 32766, 33096, 32771, 32149] avg=32768.0000 max=33096 max_rank=1 min=32149 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.663039 moved_to_replicas=133636 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[16, 28561, 64783, 11, 65507, 15116, 55377, 32773] avg=32768.0000 max=65507 max_rank=4 min=11 min_rank=3 imbalance_ratio=1.999115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[31703, 32862, 33096, 32422, 33096, 33095, 33097, 32773] avg=32768.0000 max=33097 max_rank=6 min=31703 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32410 max_load_reduction_pct=0.494756 moved_to_replicas=174993 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[25, 98219, 34321, 32783, 5, 33563, 32771, 30457] avg=32768.0000 max=98219 max_rank=1 min=5 min_rank=4 imbalance_ratio=2.997406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[25, 98219, 34321, 32783, 5, 33563, 32771, 30457] avg=32768.0000 max=98219 max_rank=1 min=5 min_rank=4 imbalance_ratio=2.997406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[31703, 32862, 33096, 32422, 33096, 33095, 33097, 32773] avg=32768.0000 max=33097 max_rank=6 min=31703 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=32410 max_load_reduction_pct=0.494756 moved_to_replicas=174993 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[25, 98219, 34321, 32783, 5, 33563, 32771, 30457] avg=32768.0000 max=98219 max_rank=1 min=5 min_rank=4 imbalance_ratio=2.997406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[39700, 65028, 70, 32733, 29491, 32779, 62337, 6] avg=32768.0000 max=65028 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.984497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[39700, 65028, 70, 32733, 29491, 32779, 62337, 6] avg=32768.0000 max=65028 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.984497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[39700, 65028, 70, 32733, 29491, 32779, 62337, 6] avg=32768.0000 max=65028 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.984497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[32387, 33096, 33096, 32783, 32766, 33096, 32771, 32149] avg=32768.0000 max=33096 max_rank=1 min=32149 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.663039 moved_to_replicas=133636 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[32387, 33096, 33096, 32783, 32766, 33096, 32771, 32149] avg=32768.0000 max=33096 max_rank=1 min=32149 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.663039 moved_to_replicas=133636 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[32387, 33096, 33096, 32783, 32766, 33096, 32771, 32149] avg=32768.0000 max=33096 max_rank=1 min=32149 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.663039 moved_to_replicas=133636 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[25, 98219, 34321, 32783, 5, 33563, 32771, 30457] avg=32768.0000 max=98219 max_rank=1 min=5 min_rank=4 imbalance_ratio=2.997406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 32733, 32310, 32779, 33096, 31938] avg=32768.0000 max=33096 max_rank=0 min=31938 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31932 max_load_reduction_pct=0.491050 moved_to_replicas=135493 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 32733, 32310, 32779, 33096, 31938] avg=32768.0000 max=33096 max_rank=0 min=31938 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31932 max_load_reduction_pct=0.491050 moved_to_replicas=135493 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 32733, 32310, 32779, 33096, 31938] avg=32768.0000 max=33096 max_rank=0 min=31938 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31932 max_load_reduction_pct=0.491050 moved_to_replicas=135493 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[25, 98219, 34321, 32783, 5, 33563, 32771, 30457] avg=32768.0000 max=98219 max_rank=1 min=5 min_rank=4 imbalance_ratio=2.997406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[32387, 33096, 33096, 32783, 32766, 33096, 32771, 32149] avg=32768.0000 max=33096 max_rank=1 min=32149 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.663039 moved_to_replicas=133636 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[39700, 65028, 70, 32733, 29491, 32779, 62337, 6] avg=32768.0000 max=65028 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.984497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[39700, 65028, 70, 32733, 29491, 32779, 62337, 6] avg=32768.0000 max=65028 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.984497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[32387, 33096, 33096, 32783, 32766, 33096, 32771, 32149] avg=32768.0000 max=33096 max_rank=1 min=32149 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65123 max_load_reduction_pct=0.663039 moved_to_replicas=133636 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[39700, 65028, 70, 32733, 29491, 32779, 62337, 6] avg=32768.0000 max=65028 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.984497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[32804, 32691, 32771, 32952, 65462, 32765, 12, 32687] avg=32768.0000 max=65462 max_rank=4 min=12 min_rank=6 imbalance_ratio=1.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[32804, 32691, 32771, 32952, 65462, 32765, 12, 32687] avg=32768.0000 max=65462 max_rank=4 min=12 min_rank=6 imbalance_ratio=1.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[32804, 32691, 32771, 32952, 65462, 32765, 12, 32687] avg=32768.0000 max=65462 max_rank=4 min=12 min_rank=6 imbalance_ratio=1.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 32733, 32310, 32779, 33096, 31938] avg=32768.0000 max=33096 max_rank=0 min=31938 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31932 max_load_reduction_pct=0.491050 moved_to_replicas=135493 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 32733, 32310, 32779, 33096, 31938] avg=32768.0000 max=33096 max_rank=0 min=31938 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31932 max_load_reduction_pct=0.491050 moved_to_replicas=135493 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[39700, 65028, 70, 32733, 29491, 32779, 62337, 6] avg=32768.0000 max=65028 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.984497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 32733, 32310, 32779, 33096, 31938] avg=32768.0000 max=33096 max_rank=0 min=31938 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31932 max_load_reduction_pct=0.491050 moved_to_replicas=135493 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[32804, 32691, 32771, 32952, 33096, 32765, 32378, 32687] avg=32768.0000 max=33096 max_rank=4 min=32378 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64770 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[32804, 32691, 32771, 32952, 33096, 32765, 32378, 32687] avg=32768.0000 max=33096 max_rank=4 min=32378 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64770 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[32804, 32691, 32771, 32952, 33096, 32765, 32378, 32687] avg=32768.0000 max=33096 max_rank=4 min=32378 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64770 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[39700, 65028, 70, 32733, 29491, 32779, 62337, 6] avg=32768.0000 max=65028 max_rank=1 min=6 min_rank=7 imbalance_ratio=1.984497 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 32733, 32310, 32779, 33096, 31938] avg=32768.0000 max=33096 max_rank=0 min=31938 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31932 max_load_reduction_pct=0.491050 moved_to_replicas=135493 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[32804, 32691, 32771, 32952, 65462, 32765, 12, 32687] avg=32768.0000 max=65462 max_rank=4 min=12 min_rank=6 imbalance_ratio=1.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[32804, 32691, 32771, 32952, 65462, 32765, 12, 32687] avg=32768.0000 max=65462 max_rank=4 min=12 min_rank=6 imbalance_ratio=1.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[33096, 33096, 33096, 32733, 32310, 32779, 33096, 31938] avg=32768.0000 max=33096 max_rank=0 min=31938 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=31932 max_load_reduction_pct=0.491050 moved_to_replicas=135493 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[32804, 32691, 32771, 32952, 65462, 32765, 12, 32687] avg=32768.0000 max=65462 max_rank=4 min=12 min_rank=6 imbalance_ratio=1.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[33042, 94, 97641, 32615, 65526, 32725, 491, 10] avg=32768.0000 max=97641 max_rank=2 min=10 min_rank=7 imbalance_ratio=2.979767 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[33042, 94, 97641, 32615, 65526, 32725, 491, 10] avg=32768.0000 max=97641 max_rank=2 min=10 min_rank=7 imbalance_ratio=2.979767 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[33042, 94, 97641, 32615, 65526, 32725, 491, 10] avg=32768.0000 max=97641 max_rank=2 min=10 min_rank=7 imbalance_ratio=2.979767 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[32804, 32691, 32771, 32952, 33096, 32765, 32378, 32687] avg=32768.0000 max=33096 max_rank=4 min=32378 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64770 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[32804, 32691, 32771, 32952, 33096, 32765, 32378, 32687] avg=32768.0000 max=33096 max_rank=4 min=32378 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64770 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[32804, 32691, 32771, 32952, 65462, 32765, 12, 32687] avg=32768.0000 max=65462 max_rank=4 min=12 min_rank=6 imbalance_ratio=1.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[32804, 32691, 32771, 32952, 33096, 32765, 32378, 32687] avg=32768.0000 max=33096 max_rank=4 min=32378 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64770 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[33042, 31886, 33096, 32615, 33096, 32725, 32921, 32763] avg=32768.0000 max=33096 max_rank=2 min=31886 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64545 max_load_reduction_pct=0.661044 moved_to_replicas=193175 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[33042, 31886, 33096, 32615, 33096, 32725, 32921, 32763] avg=32768.0000 max=33096 max_rank=2 min=31886 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64545 max_load_reduction_pct=0.661044 moved_to_replicas=193175 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[33042, 31886, 33096, 32615, 33096, 32725, 32921, 32763] avg=32768.0000 max=33096 max_rank=2 min=31886 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64545 max_load_reduction_pct=0.661044 moved_to_replicas=193175 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[32804, 32691, 32771, 32952, 65462, 32765, 12, 32687] avg=32768.0000 max=65462 max_rank=4 min=12 min_rank=6 imbalance_ratio=1.997742 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[32804, 32691, 32771, 32952, 33096, 32765, 32378, 32687] avg=32768.0000 max=33096 max_rank=4 min=32378 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64770 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[33042, 94, 97641, 32615, 65526, 32725, 491, 10] avg=32768.0000 max=97641 max_rank=2 min=10 min_rank=7 imbalance_ratio=2.979767 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[33042, 94, 97641, 32615, 65526, 32725, 491, 10] avg=32768.0000 max=97641 max_rank=2 min=10 min_rank=7 imbalance_ratio=2.979767 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[32804, 32691, 32771, 32952, 33096, 32765, 32378, 32687] avg=32768.0000 max=33096 max_rank=4 min=32378 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32366 max_load_reduction_pct=0.494424 moved_to_replicas=64770 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[33042, 94, 97641, 32615, 65526, 32725, 491, 10] avg=32768.0000 max=97641 max_rank=2 min=10 min_rank=7 imbalance_ratio=2.979767 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[1591, 32772, 98284, 13, 31298, 32773, 99, 65314] avg=32768.0000 max=98284 max_rank=2 min=13 min_rank=3 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[1591, 32772, 98284, 13, 31298, 32773, 99, 65314] avg=32768.0000 max=98284 max_rank=2 min=13 min_rank=3 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[1591, 32772, 98284, 13, 31298, 32773, 99, 65314] avg=32768.0000 max=98284 max_rank=2 min=13 min_rank=3 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[33042, 31886, 33096, 32615, 33096, 32725, 32921, 32763] avg=32768.0000 max=33096 max_rank=2 min=31886 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64545 max_load_reduction_pct=0.661044 moved_to_replicas=193175 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[33042, 31886, 33096, 32615, 33096, 32725, 32921, 32763] avg=32768.0000 max=33096 max_rank=2 min=31886 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64545 max_load_reduction_pct=0.661044 moved_to_replicas=193175 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[33042, 94, 97641, 32615, 65526, 32725, 491, 10] avg=32768.0000 max=97641 max_rank=2 min=10 min_rank=7 imbalance_ratio=2.979767 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32772, 33096, 32774, 32011, 32773, 32526, 33096] avg=32768.0000 max=33096 max_rank=0 min=32011 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=194952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[33042, 31886, 33096, 32615, 33096, 32725, 32921, 32763] avg=32768.0000 max=33096 max_rank=2 min=31886 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64545 max_load_reduction_pct=0.661044 moved_to_replicas=193175 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32772, 33096, 32774, 32011, 32773, 32526, 33096] avg=32768.0000 max=33096 max_rank=0 min=32011 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=194952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[33042, 94, 97641, 32615, 65526, 32725, 491, 10] avg=32768.0000 max=97641 max_rank=2 min=10 min_rank=7 imbalance_ratio=2.979767 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32772, 33096, 32774, 32011, 32773, 32526, 33096] avg=32768.0000 max=33096 max_rank=0 min=32011 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=194952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[33042, 31886, 33096, 32615, 33096, 32725, 32921, 32763] avg=32768.0000 max=33096 max_rank=2 min=31886 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64545 max_load_reduction_pct=0.661044 moved_to_replicas=193175 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[1591, 32772, 98284, 13, 31298, 32773, 99, 65314] avg=32768.0000 max=98284 max_rank=2 min=13 min_rank=3 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[1591, 32772, 98284, 13, 31298, 32773, 99, 65314] avg=32768.0000 max=98284 max_rank=2 min=13 min_rank=3 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[33042, 31886, 33096, 32615, 33096, 32725, 32921, 32763] avg=32768.0000 max=33096 max_rank=2 min=31886 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64545 max_load_reduction_pct=0.661044 moved_to_replicas=193175 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[14, 8, 2302, 29755, 38165, 55422, 38368, 98110] avg=32768.0000 max=98110 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.994080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[1591, 32772, 98284, 13, 31298, 32773, 99, 65314] avg=32768.0000 max=98284 max_rank=2 min=13 min_rank=3 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[14, 8, 2302, 29755, 38165, 55422, 38368, 98110] avg=32768.0000 max=98110 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.994080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32772, 33096, 32774, 32011, 32773, 32526, 33096] avg=32768.0000 max=33096 max_rank=0 min=32011 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=194952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[14, 8, 2302, 29755, 38165, 55422, 38368, 98110] avg=32768.0000 max=98110 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.994080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32772, 33096, 32774, 32011, 32773, 32526, 33096] avg=32768.0000 max=33096 max_rank=0 min=32011 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=194952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[1591, 32772, 98284, 13, 31298, 32773, 99, 65314] avg=32768.0000 max=98284 max_rank=2 min=13 min_rank=3 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[32272, 32764, 31627, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31627 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=65013 max_load_reduction_pct=0.662654 moved_to_replicas=194913 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32772, 33096, 32774, 32011, 32773, 32526, 33096] avg=32768.0000 max=33096 max_rank=0 min=32011 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=194952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[1591, 32772, 98284, 13, 31298, 32773, 99, 65314] avg=32768.0000 max=98284 max_rank=2 min=13 min_rank=3 imbalance_ratio=2.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[32272, 32764, 31627, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31627 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=65013 max_load_reduction_pct=0.662654 moved_to_replicas=194913 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[32272, 32764, 31627, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31627 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=65013 max_load_reduction_pct=0.662654 moved_to_replicas=194913 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32772, 33096, 32774, 32011, 32773, 32526, 33096] avg=32768.0000 max=33096 max_rank=0 min=32011 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=194952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[14, 8, 2302, 29755, 38165, 55422, 38368, 98110] avg=32768.0000 max=98110 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.994080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[14, 8, 2302, 29755, 38165, 55422, 38368, 98110] avg=32768.0000 max=98110 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.994080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32772, 33096, 32774, 32011, 32773, 32526, 33096] avg=32768.0000 max=33096 max_rank=0 min=32011 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65188 max_load_reduction_pct=0.663262 moved_to_replicas=194952 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[65450, 8, 16, 340, 65528, 32518, 65527, 32757] avg=32768.0000 max=65528 max_rank=4 min=8 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[14, 8, 2302, 29755, 38165, 55422, 38368, 98110] avg=32768.0000 max=98110 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.994080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[65450, 8, 16, 340, 65528, 32518, 65527, 32757] avg=32768.0000 max=65528 max_rank=4 min=8 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[32272, 32764, 31627, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31627 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=65013 max_load_reduction_pct=0.662654 moved_to_replicas=194913 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[65450, 8, 16, 340, 65528, 32518, 65527, 32757] avg=32768.0000 max=65528 max_rank=4 min=8 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[32272, 32764, 31627, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31627 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=65013 max_load_reduction_pct=0.662654 moved_to_replicas=194913 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[14, 8, 2302, 29755, 38165, 55422, 38368, 98110] avg=32768.0000 max=98110 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.994080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[33096, 32440, 32447, 32694, 33096, 32518, 33096, 32757] avg=32768.0000 max=33096 max_rank=0 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194458 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[33096, 32440, 32447, 32694, 33096, 32518, 33096, 32757] avg=32768.0000 max=33096 max_rank=0 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194458 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[32272, 32764, 31627, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31627 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=65013 max_load_reduction_pct=0.662654 moved_to_replicas=194913 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[14, 8, 2302, 29755, 38165, 55422, 38368, 98110] avg=32768.0000 max=98110 max_rank=7 min=8 min_rank=1 imbalance_ratio=2.994080 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[33096, 32440, 32447, 32694, 33096, 32518, 33096, 32757] avg=32768.0000 max=33096 max_rank=0 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194458 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[32272, 32764, 31627, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31627 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=65013 max_load_reduction_pct=0.662654 moved_to_replicas=194913 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[65450, 8, 16, 340, 65528, 32518, 65527, 32757] avg=32768.0000 max=65528 max_rank=4 min=8 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[65450, 8, 16, 340, 65528, 32518, 65527, 32757] avg=32768.0000 max=65528 max_rank=4 min=8 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[32601, 65487, 36340, 65540, 30247, 16, 29, 31884] avg=32768.0000 max=65540 max_rank=3 min=16 min_rank=5 imbalance_ratio=2.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[32272, 32764, 31627, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31627 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=65013 max_load_reduction_pct=0.662654 moved_to_replicas=194913 active_clone_links=12 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[32601, 65487, 36340, 65540, 30247, 16, 29, 31884] avg=32768.0000 max=65540 max_rank=3 min=16 min_rank=5 imbalance_ratio=2.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[33096, 32440, 32447, 32694, 33096, 32518, 33096, 32757] avg=32768.0000 max=33096 max_rank=0 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194458 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[33096, 32440, 32447, 32694, 33096, 32518, 33096, 32757] avg=32768.0000 max=33096 max_rank=0 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194458 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[65450, 8, 16, 340, 65528, 32518, 65527, 32757] avg=32768.0000 max=65528 max_rank=4 min=8 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[32601, 65487, 36340, 65540, 30247, 16, 29, 31884] avg=32768.0000 max=65540 max_rank=3 min=16 min_rank=5 imbalance_ratio=2.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[65450, 8, 16, 340, 65528, 32518, 65527, 32757] avg=32768.0000 max=65528 max_rank=4 min=8 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[32601, 33096, 33096, 33096, 33096, 32460, 32420, 32279] avg=32768.0000 max=33096 max_rank=1 min=32279 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32444 max_load_reduction_pct=0.495026 moved_to_replicas=136061 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[32601, 33096, 33096, 33096, 33096, 32460, 32420, 32279] avg=32768.0000 max=33096 max_rank=1 min=32279 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32444 max_load_reduction_pct=0.495026 moved_to_replicas=136061 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[33096, 32440, 32447, 32694, 33096, 32518, 33096, 32757] avg=32768.0000 max=33096 max_rank=0 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194458 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[33096, 32440, 32447, 32694, 33096, 32518, 33096, 32757] avg=32768.0000 max=33096 max_rank=0 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194458 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[32601, 33096, 33096, 33096, 33096, 32460, 32420, 32279] avg=32768.0000 max=33096 max_rank=1 min=32279 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32444 max_load_reduction_pct=0.495026 moved_to_replicas=136061 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[65450, 8, 16, 340, 65528, 32518, 65527, 32757] avg=32768.0000 max=65528 max_rank=4 min=8 min_rank=1 imbalance_ratio=1.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[32601, 65487, 36340, 65540, 30247, 16, 29, 31884] avg=32768.0000 max=65540 max_rank=3 min=16 min_rank=5 imbalance_ratio=2.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[32601, 65487, 36340, 65540, 30247, 16, 29, 31884] avg=32768.0000 max=65540 max_rank=3 min=16 min_rank=5 imbalance_ratio=2.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[58, 32800, 65516, 32742, 32756, 32507, 33011, 32754] avg=32768.0000 max=65516 max_rank=2 min=58 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[58, 32800, 65516, 32742, 32756, 32507, 33011, 32754] avg=32768.0000 max=65516 max_rank=2 min=58 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[32601, 33096, 33096, 33096, 33096, 32460, 32420, 32279] avg=32768.0000 max=33096 max_rank=1 min=32279 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32444 max_load_reduction_pct=0.495026 moved_to_replicas=136061 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[33096, 32440, 32447, 32694, 33096, 32518, 33096, 32757] avg=32768.0000 max=33096 max_rank=0 min=32440 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32432 max_load_reduction_pct=0.494933 moved_to_replicas=194458 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[32601, 33096, 33096, 33096, 33096, 32460, 32420, 32279] avg=32768.0000 max=33096 max_rank=1 min=32279 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32444 max_load_reduction_pct=0.495026 moved_to_replicas=136061 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[32601, 65487, 36340, 65540, 30247, 16, 29, 31884] avg=32768.0000 max=65540 max_rank=3 min=16 min_rank=5 imbalance_ratio=2.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[32601, 65487, 36340, 65540, 30247, 16, 29, 31884] avg=32768.0000 max=65540 max_rank=3 min=16 min_rank=5 imbalance_ratio=2.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[58, 32800, 65516, 32742, 32756, 32507, 33011, 32754] avg=32768.0000 max=65516 max_rank=2 min=58 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[32478, 32800, 33096, 32742, 32756, 32507, 33011, 32754] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=64850 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[32478, 32800, 33096, 32742, 32756, 32507, 33011, 32754] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=64850 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[32601, 33096, 33096, 33096, 33096, 32460, 32420, 32279] avg=32768.0000 max=33096 max_rank=1 min=32279 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32444 max_load_reduction_pct=0.495026 moved_to_replicas=136061 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[32601, 33096, 33096, 33096, 33096, 32460, 32420, 32279] avg=32768.0000 max=33096 max_rank=1 min=32279 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32444 max_load_reduction_pct=0.495026 moved_to_replicas=136061 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[32478, 32800, 33096, 32742, 32756, 32507, 33011, 32754] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=64850 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[58, 32800, 65516, 32742, 32756, 32507, 33011, 32754] avg=32768.0000 max=65516 max_rank=2 min=58 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[32601, 65487, 36340, 65540, 30247, 16, 29, 31884] avg=32768.0000 max=65540 max_rank=3 min=16 min_rank=5 imbalance_ratio=2.000122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[58, 32800, 65516, 32742, 32756, 32507, 33011, 32754] avg=32768.0000 max=65516 max_rank=2 min=58 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[37, 151, 71237, 13, 92373, 32783, 32765, 32785] avg=32768.0000 max=92373 max_rank=4 min=13 min_rank=3 imbalance_ratio=2.819000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[37, 151, 71237, 13, 92373, 32783, 32765, 32785] avg=32768.0000 max=92373 max_rank=4 min=13 min_rank=3 imbalance_ratio=2.819000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[32478, 32800, 33096, 32742, 32756, 32507, 33011, 32754] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=64850 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[32478, 32800, 33096, 32742, 32756, 32507, 33011, 32754] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=64850 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[32601, 33096, 33096, 33096, 33096, 32460, 32420, 32279] avg=32768.0000 max=33096 max_rank=1 min=32279 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32444 max_load_reduction_pct=0.495026 moved_to_replicas=136061 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[58, 32800, 65516, 32742, 32756, 32507, 33011, 32754] avg=32768.0000 max=65516 max_rank=2 min=58 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[58, 32800, 65516, 32742, 32756, 32507, 33011, 32754] avg=32768.0000 max=65516 max_rank=2 min=58 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[37, 151, 71237, 13, 92373, 32783, 32765, 32785] avg=32768.0000 max=92373 max_rank=4 min=13 min_rank=3 imbalance_ratio=2.819000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[31950, 32907, 33096, 32764, 33094, 32783, 32765, 32785] avg=32768.0000 max=33096 max_rank=2 min=31950 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=59277 max_load_reduction_pct=0.641713 moved_to_replicas=194910 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[31950, 32907, 33096, 32764, 33094, 32783, 32765, 32785] avg=32768.0000 max=33096 max_rank=2 min=31950 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=59277 max_load_reduction_pct=0.641713 moved_to_replicas=194910 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[32478, 32800, 33096, 32742, 32756, 32507, 33011, 32754] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=64850 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[32478, 32800, 33096, 32742, 32756, 32507, 33011, 32754] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=64850 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[31950, 32907, 33096, 32764, 33094, 32783, 32765, 32785] avg=32768.0000 max=33096 max_rank=2 min=31950 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=59277 max_load_reduction_pct=0.641713 moved_to_replicas=194910 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[37, 151, 71237, 13, 92373, 32783, 32765, 32785] avg=32768.0000 max=92373 max_rank=4 min=13 min_rank=3 imbalance_ratio=2.819000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[37, 151, 71237, 13, 92373, 32783, 32765, 32785] avg=32768.0000 max=92373 max_rank=4 min=13 min_rank=3 imbalance_ratio=2.819000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[58, 32800, 65516, 32742, 32756, 32507, 33011, 32754] avg=32768.0000 max=65516 max_rank=2 min=58 min_rank=0 imbalance_ratio=1.999390 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[15, 33049, 50, 32770, 65919, 32777, 32589, 64975] avg=32768.0000 max=65919 max_rank=4 min=15 min_rank=0 imbalance_ratio=2.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[15, 33049, 50, 32770, 65919, 32777, 32589, 64975] avg=32768.0000 max=65919 max_rank=4 min=15 min_rank=0 imbalance_ratio=2.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[31950, 32907, 33096, 32764, 33094, 32783, 32765, 32785] avg=32768.0000 max=33096 max_rank=2 min=31950 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=59277 max_load_reduction_pct=0.641713 moved_to_replicas=194910 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[31950, 32907, 33096, 32764, 33094, 32783, 32765, 32785] avg=32768.0000 max=33096 max_rank=2 min=31950 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=59277 max_load_reduction_pct=0.641713 moved_to_replicas=194910 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[37, 151, 71237, 13, 92373, 32783, 32765, 32785] avg=32768.0000 max=92373 max_rank=4 min=13 min_rank=3 imbalance_ratio=2.819000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[32478, 32800, 33096, 32742, 32756, 32507, 33011, 32754] avg=32768.0000 max=33096 max_rank=2 min=32478 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32420 max_load_reduction_pct=0.494841 moved_to_replicas=64850 active_clone_links=2 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[37, 151, 71237, 13, 92373, 32783, 32765, 32785] avg=32768.0000 max=92373 max_rank=4 min=13 min_rank=3 imbalance_ratio=2.819000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[15, 33049, 50, 32770, 65919, 32777, 32589, 64975] avg=32768.0000 max=65919 max_rank=4 min=15 min_rank=0 imbalance_ratio=2.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[32763, 33049, 32004, 32770, 33096, 32777, 32589, 33096] avg=32768.0000 max=33096 max_rank=4 min=32004 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32823 max_load_reduction_pct=0.497929 moved_to_replicas=129411 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[32763, 33049, 32004, 32770, 33096, 32777, 32589, 33096] avg=32768.0000 max=33096 max_rank=4 min=32004 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32823 max_load_reduction_pct=0.497929 moved_to_replicas=129411 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[31950, 32907, 33096, 32764, 33094, 32783, 32765, 32785] avg=32768.0000 max=33096 max_rank=2 min=31950 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=59277 max_load_reduction_pct=0.641713 moved_to_replicas=194910 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[15, 33049, 50, 32770, 65919, 32777, 32589, 64975] avg=32768.0000 max=65919 max_rank=4 min=15 min_rank=0 imbalance_ratio=2.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[31950, 32907, 33096, 32764, 33094, 32783, 32765, 32785] avg=32768.0000 max=33096 max_rank=2 min=31950 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=59277 max_load_reduction_pct=0.641713 moved_to_replicas=194910 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[32763, 33049, 32004, 32770, 33096, 32777, 32589, 33096] avg=32768.0000 max=33096 max_rank=4 min=32004 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32823 max_load_reduction_pct=0.497929 moved_to_replicas=129411 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[15, 33049, 50, 32770, 65919, 32777, 32589, 64975] avg=32768.0000 max=65919 max_rank=4 min=15 min_rank=0 imbalance_ratio=2.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[37, 151, 71237, 13, 92373, 32783, 32765, 32785] avg=32768.0000 max=92373 max_rank=4 min=13 min_rank=3 imbalance_ratio=2.819000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[97693, 32206, 64882, 32765, 18, 1811, 32768, 1] avg=32768.0000 max=97693 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.981354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[97693, 32206, 64882, 32765, 18, 1811, 32768, 1] avg=32768.0000 max=97693 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.981354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[32763, 33049, 32004, 32770, 33096, 32777, 32589, 33096] avg=32768.0000 max=33096 max_rank=4 min=32004 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32823 max_load_reduction_pct=0.497929 moved_to_replicas=129411 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[32763, 33049, 32004, 32770, 33096, 32777, 32589, 33096] avg=32768.0000 max=33096 max_rank=4 min=32004 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32823 max_load_reduction_pct=0.497929 moved_to_replicas=129411 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[15, 33049, 50, 32770, 65919, 32777, 32589, 64975] avg=32768.0000 max=65919 max_rank=4 min=15 min_rank=0 imbalance_ratio=2.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[31950, 32907, 33096, 32764, 33094, 32783, 32765, 32785] avg=32768.0000 max=33096 max_rank=2 min=31950 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=59277 max_load_reduction_pct=0.641713 moved_to_replicas=194910 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[33096, 32206, 33096, 32765, 32423, 33096, 32768, 32694] avg=32768.0000 max=33096 max_rank=0 min=32206 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64597 max_load_reduction_pct=0.661224 moved_to_replicas=193011 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[15, 33049, 50, 32770, 65919, 32777, 32589, 64975] avg=32768.0000 max=65919 max_rank=4 min=15 min_rank=0 imbalance_ratio=2.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[97693, 32206, 64882, 32765, 18, 1811, 32768, 1] avg=32768.0000 max=97693 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.981354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[33096, 32206, 33096, 32765, 32423, 33096, 32768, 32694] avg=32768.0000 max=33096 max_rank=0 min=32206 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64597 max_load_reduction_pct=0.661224 moved_to_replicas=193011 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[32763, 33049, 32004, 32770, 33096, 32777, 32589, 33096] avg=32768.0000 max=33096 max_rank=4 min=32004 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32823 max_load_reduction_pct=0.497929 moved_to_replicas=129411 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[97693, 32206, 64882, 32765, 18, 1811, 32768, 1] avg=32768.0000 max=97693 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.981354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[32763, 33049, 32004, 32770, 33096, 32777, 32589, 33096] avg=32768.0000 max=33096 max_rank=4 min=32004 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32823 max_load_reduction_pct=0.497929 moved_to_replicas=129411 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[97693, 32206, 64882, 32765, 18, 1811, 32768, 1] avg=32768.0000 max=97693 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.981354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[33096, 32206, 33096, 32765, 32423, 33096, 32768, 32694] avg=32768.0000 max=33096 max_rank=0 min=32206 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64597 max_load_reduction_pct=0.661224 moved_to_replicas=193011 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[15, 33049, 50, 32770, 65919, 32777, 32589, 64975] avg=32768.0000 max=65919 max_rank=4 min=15 min_rank=0 imbalance_ratio=2.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[33096, 32206, 33096, 32765, 32423, 33096, 32768, 32694] avg=32768.0000 max=33096 max_rank=0 min=32206 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64597 max_load_reduction_pct=0.661224 moved_to_replicas=193011 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[33096, 32206, 33096, 32765, 32423, 33096, 32768, 32694] avg=32768.0000 max=33096 max_rank=0 min=32206 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64597 max_load_reduction_pct=0.661224 moved_to_replicas=193011 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[97693, 32206, 64882, 32765, 18, 1811, 32768, 1] avg=32768.0000 max=97693 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.981354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[32763, 33049, 32004, 32770, 33096, 32777, 32589, 33096] avg=32768.0000 max=33096 max_rank=4 min=32004 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32823 max_load_reduction_pct=0.497929 moved_to_replicas=129411 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[97693, 32206, 64882, 32765, 18, 1811, 32768, 1] avg=32768.0000 max=97693 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.981354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[33096, 32206, 33096, 32765, 32423, 33096, 32768, 32694] avg=32768.0000 max=33096 max_rank=0 min=32206 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64597 max_load_reduction_pct=0.661224 moved_to_replicas=193011 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[33096, 32206, 33096, 32765, 32423, 33096, 32768, 32694] avg=32768.0000 max=33096 max_rank=0 min=32206 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64597 max_load_reduction_pct=0.661224 moved_to_replicas=193011 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[97693, 32206, 64882, 32765, 18, 1811, 32768, 1] avg=32768.0000 max=97693 max_rank=0 min=1 min_rank=7 imbalance_ratio=2.981354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[33096, 32206, 33096, 32765, 32423, 33096, 32768, 32694] avg=32768.0000 max=33096 max_rank=0 min=32206 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=64597 max_load_reduction_pct=0.661224 moved_to_replicas=193011 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=32 total_expected_recv=1050997 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=250624 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=32 total_expected_recv=1046951 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253312 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=32 total_expected_recv=1053018 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=246016 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=32 total_expected_recv=1046774 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=255232 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=32 total_expected_recv=1049929 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=252032 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=32 total_expected_recv=1047754 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=253056 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=32 total_expected_recv=1041523 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=257024 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=32 total_expected_recv=1051662 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=249088 peak_demand=33096 required_capacity=1.010010 + [2026-08-30 05:10:31] iteration 10/ 12 | consumed samples: 160 | elapsed time per iteration (ms): 563.1 | throughput per GPU (TFLOP/s/GPU): 592.0 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.105211E+00 | loss scale: 1.0 | grad norm: 5.515 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=40960 expected_recv=64927 expected_overflow=0 exact_recv_per_rank=[33096, 31471, 33095, 33096, 32099, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31471, 33095, 33096, 32099, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=40960 expected_recv=64567 expected_overflow=0 exact_recv_per_rank=[33096, 31471, 33095, 33096, 32099, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=40960 expected_recv=64832 expected_overflow=0 exact_recv_per_rank=[33096, 31471, 33095, 33096, 32099, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 31471, 33095, 33096, 32099, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31471, 33095, 33096, 32099, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 31471, 33095, 33096, 32099, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=2 budget=40960 expected_recv=64927 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=2 budget=40960 expected_recv=64567 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=2 budget=40960 expected_recv=64832 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=40960 expected_recv=65194 expected_overflow=0 exact_recv_per_rank=[33096, 31471, 33095, 33096, 32099, 33095, 33095, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=2 budget=40960 expected_recv=65194 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65539 expected_overflow=0 exact_recv_per_rank=[33096, 32681, 33096, 33096, 32765, 32765, 32779, 31866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65539 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32681, 33096, 33096, 32765, 32765, 32779, 31866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=40960 expected_recv=64962 expected_overflow=0 exact_recv_per_rank=[33096, 32681, 33096, 33096, 32765, 32765, 32779, 31866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65352 expected_overflow=0 exact_recv_per_rank=[33096, 32681, 33096, 33096, 32765, 32765, 32779, 31866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65352 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32681, 33096, 33096, 32765, 32765, 32779, 31866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=2 budget=40960 expected_recv=64962 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65557 expected_overflow=0 exact_recv_per_rank=[33096, 32681, 33096, 33096, 32765, 32765, 32779, 31866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=40960 expected_recv=64962 expected_overflow=0 exact_recv_per_rank=[33096, 32681, 33096, 33096, 32765, 32765, 32779, 31866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[33096, 32681, 33096, 33096, 32765, 32765, 32779, 31866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65557 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=2 budget=40960 expected_recv=64962 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=2 budget=40960 expected_recv=65532 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32791, 31684, 33095, 32768, 32768, 33096, 33095, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65599 expected_overflow=0 exact_recv_per_rank=[32791, 31684, 33095, 32768, 32768, 33096, 33095, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[32791, 31684, 33095, 32768, 32768, 33096, 33095, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=40960 expected_recv=63107 expected_overflow=0 exact_recv_per_rank=[32791, 31684, 33095, 32768, 32768, 33096, 33095, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65599 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32791, 31684, 33095, 32768, 32768, 33096, 33095, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65535 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=2 budget=40960 expected_recv=63107 actual_valid_recv=63872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65943 expected_overflow=0 exact_recv_per_rank=[32791, 31684, 33095, 32768, 32768, 33096, 33095, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32791, 31684, 33095, 32768, 32768, 33096, 33095, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=63872 slack_slots=18048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65943 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[32791, 31684, 33095, 32768, 32768, 33096, 33095, 32847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=40960 expected_recv=64553 expected_overflow=0 exact_recv_per_rank=[33095, 32664, 32187, 32745, 33096, 32165, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=2 budget=40960 expected_recv=65529 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=2 budget=40960 expected_recv=64553 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33095, 32664, 32187, 32745, 33096, 32165, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=40960 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[33095, 32664, 32187, 32745, 33096, 32165, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=40960 expected_recv=65474 expected_overflow=0 exact_recv_per_rank=[33095, 32664, 32187, 32745, 33096, 32165, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=40960 expected_recv=63946 expected_overflow=0 exact_recv_per_rank=[33095, 32664, 32187, 32745, 33096, 32165, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 32664, 32187, 32745, 33096, 32165, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=2 budget=40960 expected_recv=65546 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=2 budget=40960 expected_recv=65474 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 32664, 32187, 32745, 33096, 32165, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=2 budget=40960 expected_recv=63946 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65559 expected_overflow=0 exact_recv_per_rank=[31538, 33096, 33096, 32793, 33096, 32762, 32768, 32995] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 32664, 32187, 32745, 33096, 32165, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65559 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=40960 expected_recv=63169 expected_overflow=0 exact_recv_per_rank=[31538, 33096, 33096, 32793, 33096, 32762, 32768, 32995] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31538, 33096, 33096, 32793, 33096, 32762, 32768, 32995] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=2 budget=40960 expected_recv=63169 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65562 expected_overflow=0 exact_recv_per_rank=[31538, 33096, 33096, 32793, 33096, 32762, 32768, 32995] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31538, 33096, 33096, 32793, 33096, 32762, 32768, 32995] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65881 expected_overflow=0 exact_recv_per_rank=[31538, 33096, 33096, 32793, 33096, 32762, 32768, 32995] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65562 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[31538, 33096, 33096, 32793, 33096, 32762, 32768, 32995] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32848, 31378, 32768, 33096, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65881 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=2 budget=40960 expected_recv=65541 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31538, 33096, 33096, 32793, 33096, 32762, 32768, 32995] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65535 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32848, 31378, 32768, 33096, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32848, 31378, 32768, 33096, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65595 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32848, 31378, 32768, 33096, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32848, 31378, 32768, 33096, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32848, 31378, 32768, 33096, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32848, 31378, 32768, 33096, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65595 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65543 expected_overflow=0 exact_recv_per_rank=[32768, 32741, 32508, 33096, 33096, 32768, 32405, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=2 budget=40960 expected_recv=65536 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=40960 expected_recv=62854 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 32848, 31378, 32768, 33096, 32767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65543 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=2 budget=40960 expected_recv=62854 actual_valid_recv=63872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65546 expected_overflow=0 exact_recv_per_rank=[32768, 32741, 32508, 33096, 33096, 32768, 32405, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=63872 slack_slots=18048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65546 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65492 expected_overflow=0 exact_recv_per_rank=[32768, 32741, 32508, 33096, 33096, 32768, 32405, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32768, 32741, 32508, 33096, 33096, 32768, 32405, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=40960 expected_recv=64982 expected_overflow=0 exact_recv_per_rank=[32768, 32741, 32508, 33096, 33096, 32768, 32405, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65508 expected_overflow=0 exact_recv_per_rank=[32768, 32741, 32508, 33096, 33096, 32768, 32405, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65492 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=40960 expected_recv=64833 expected_overflow=0 exact_recv_per_rank=[32768, 32741, 32508, 33096, 33096, 32768, 32405, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=2 budget=40960 expected_recv=64982 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65427 expected_overflow=0 exact_recv_per_rank=[33096, 32436, 33096, 32464, 33096, 32760, 32752, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=2 budget=40960 expected_recv=65508 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=2 budget=40960 expected_recv=64833 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65427 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32768, 32741, 32508, 33096, 33096, 32768, 32405, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32436, 33096, 32464, 33096, 32760, 32752, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=40960 expected_recv=64869 expected_overflow=0 exact_recv_per_rank=[33096, 32436, 33096, 32464, 33096, 32760, 32752, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65020 expected_overflow=0 exact_recv_per_rank=[33096, 32436, 33096, 32464, 33096, 32760, 32752, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=2 budget=40960 expected_recv=64869 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65020 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32436, 33096, 32464, 33096, 32760, 32752, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=40960 expected_recv=64895 expected_overflow=0 exact_recv_per_rank=[33096, 32436, 33096, 32464, 33096, 32760, 32752, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65501 expected_overflow=0 exact_recv_per_rank=[33096, 32436, 33096, 32464, 33096, 32760, 32752, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=40960 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[33095, 31402, 33096, 32823, 33096, 32769, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=2 budget=40960 expected_recv=64895 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=2 budget=40960 expected_recv=65501 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=2 budget=40960 expected_recv=65532 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32436, 33096, 32464, 33096, 32760, 32752, 32444] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33095, 31402, 33096, 32823, 33096, 32769, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=40960 expected_recv=62519 expected_overflow=0 exact_recv_per_rank=[33095, 31402, 33096, 32823, 33096, 32769, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=2 budget=40960 expected_recv=62519 actual_valid_recv=63232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=40960 expected_recv=65920 expected_overflow=0 exact_recv_per_rank=[33095, 31402, 33096, 32823, 33096, 32769, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=63232 slack_slots=18688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=2 budget=40960 expected_recv=65920 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=40960 expected_recv=66189 expected_overflow=0 exact_recv_per_rank=[33095, 31402, 33096, 32823, 33096, 32769, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=40960 expected_recv=65552 expected_overflow=0 exact_recv_per_rank=[33095, 31402, 33096, 32823, 33096, 32769, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 31402, 33096, 32823, 33096, 32769, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=40960 expected_recv=66164 expected_overflow=0 exact_recv_per_rank=[32398, 32768, 32028, 32899, 33096, 33068, 32791, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=2 budget=40960 expected_recv=66189 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=2 budget=40960 expected_recv=65552 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=2 budget=40960 expected_recv=66164 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 31402, 33096, 32823, 33096, 32769, 32768, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65176 expected_overflow=0 exact_recv_per_rank=[32398, 32768, 32028, 32899, 33096, 33068, 32791, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65176 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=40960 expected_recv=64877 expected_overflow=0 exact_recv_per_rank=[32398, 32768, 32028, 32899, 33096, 33068, 32791, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65995 expected_overflow=0 exact_recv_per_rank=[32398, 32768, 32028, 32899, 33096, 33068, 32791, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=2 budget=40960 expected_recv=64877 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65995 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32398, 32768, 32028, 32899, 33096, 33068, 32791, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=40960 expected_recv=64562 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 32237, 33096, 33096, 32320, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65670 expected_overflow=0 exact_recv_per_rank=[32398, 32768, 32028, 32899, 33096, 33068, 32791, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=40960 expected_recv=64022 expected_overflow=0 exact_recv_per_rank=[32398, 32768, 32028, 32899, 33096, 33068, 32791, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=2 budget=40960 expected_recv=65670 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=2 budget=40960 expected_recv=64562 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=2 budget=40960 expected_recv=64022 actual_valid_recv=64640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32398, 32768, 32028, 32899, 33096, 33068, 32791, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 32237, 33096, 33096, 32320, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=64640 slack_slots=17280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=40960 expected_recv=64891 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 32237, 33096, 33096, 32320, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 32237, 33096, 33096, 32320, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=2 budget=40960 expected_recv=64891 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=40960 expected_recv=65518 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 32237, 33096, 33096, 32320, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 32237, 33096, 33096, 32320, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=2 budget=40960 expected_recv=65518 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65538 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33096, 33096, 32622, 32768, 32260, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=40960 expected_recv=64549 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 32237, 33096, 33096, 32320, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65538 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32442, 32237, 33096, 33096, 32320, 33096, 32761] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65381 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33096, 33096, 32622, 32768, 32260, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33096, 33096, 32622, 32768, 32260, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=2 budget=40960 expected_recv=64549 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65381 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33096, 33096, 32622, 32768, 32260, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65022 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33096, 33096, 32622, 32768, 32260, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65022 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=40960 expected_recv=64605 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33096, 33096, 32622, 32768, 32260, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=40960 expected_recv=64345 expected_overflow=0 exact_recv_per_rank=[32771, 32745, 33096, 32771, 32767, 32229, 33008, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=2 budget=40960 expected_recv=64605 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=2 budget=40960 expected_recv=64345 actual_valid_recv=64768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33096, 33096, 32622, 32768, 32260, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65166 expected_overflow=0 exact_recv_per_rank=[32723, 33096, 33096, 33096, 32622, 32768, 32260, 32483] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65509 expected_overflow=0 exact_recv_per_rank=[32771, 32745, 33096, 32771, 32767, 32229, 33008, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65566 expected_overflow=0 exact_recv_per_rank=[32771, 32745, 33096, 32771, 32767, 32229, 33008, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=64768 slack_slots=17152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=2 budget=40960 expected_recv=65166 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65509 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65566 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65553 expected_overflow=0 exact_recv_per_rank=[32771, 32745, 33096, 32771, 32767, 32229, 33008, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65493 expected_overflow=0 exact_recv_per_rank=[32771, 32745, 33096, 32771, 32767, 32229, 33008, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65553 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65493 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=40960 expected_recv=66104 expected_overflow=0 exact_recv_per_rank=[32771, 32745, 33096, 32771, 32767, 32229, 33008, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=40960 expected_recv=65545 expected_overflow=0 exact_recv_per_rank=[33096, 31788, 33096, 32436, 33096, 32769, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=2 budget=40960 expected_recv=66104 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32771, 32745, 33096, 32771, 32767, 32229, 33008, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=2 budget=40960 expected_recv=65545 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65526 expected_overflow=0 exact_recv_per_rank=[32771, 32745, 33096, 32771, 32767, 32229, 33008, 32757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=40960 expected_recv=63578 expected_overflow=0 exact_recv_per_rank=[33096, 31788, 33096, 32436, 33096, 32769, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31788, 33096, 32436, 33096, 32769, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=12 calls=2 budget=40960 expected_recv=65526 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=13 calls=2 budget=40960 expected_recv=63578 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=40960 expected_recv=64863 expected_overflow=0 exact_recv_per_rank=[33096, 31788, 33096, 32436, 33096, 32769, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31788, 33096, 32436, 33096, 32769, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=2 budget=40960 expected_recv=64863 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=40960 expected_recv=65534 expected_overflow=0 exact_recv_per_rank=[33096, 31788, 33096, 32436, 33096, 32769, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[32632, 33096, 32289, 32766, 33096, 32769, 32400, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=2 budget=40960 expected_recv=65534 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65541 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31788, 33096, 32436, 33096, 32769, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31788, 33096, 32436, 33096, 32769, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32632, 33096, 32289, 32766, 33096, 32769, 32400, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65396 expected_overflow=0 exact_recv_per_rank=[32632, 33096, 32289, 32766, 33096, 32769, 32400, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65396 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[32632, 33096, 32289, 32766, 33096, 32769, 32400, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32632, 33096, 32289, 32766, 33096, 32769, 32400, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=2 budget=40960 expected_recv=65535 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=40960 expected_recv=64839 expected_overflow=0 exact_recv_per_rank=[32632, 33096, 32289, 32766, 33096, 32769, 32400, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65288 expected_overflow=0 exact_recv_per_rank=[33096, 32751, 32892, 32767, 32757, 32741, 32768, 32372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=2 budget=40960 expected_recv=64839 actual_valid_recv=65024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65288 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65024 slack_slots=16896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=40960 expected_recv=64401 expected_overflow=0 exact_recv_per_rank=[32632, 33096, 32289, 32766, 33096, 32769, 32400, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32632, 33096, 32289, 32766, 33096, 32769, 32400, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65456 expected_overflow=0 exact_recv_per_rank=[33096, 32751, 32892, 32767, 32757, 32741, 32768, 32372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32751, 32892, 32767, 32757, 32741, 32768, 32372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=14 calls=2 budget=40960 expected_recv=64401 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65456 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65110 expected_overflow=0 exact_recv_per_rank=[33096, 32751, 32892, 32767, 32757, 32741, 32768, 32372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65532 expected_overflow=0 exact_recv_per_rank=[33096, 32751, 32892, 32767, 32757, 32741, 32768, 32372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65110 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65532 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65536 expected_overflow=0 exact_recv_per_rank=[33096, 32751, 32892, 32767, 32757, 32741, 32768, 32372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[36995, 30681, 24523, 26017, 5182, 27094, 43042, 68610] avg=32768.0000 max=68610 max_rank=7 min=5182 min_rank=4 imbalance_ratio=2.093811 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65536 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65988 expected_overflow=0 exact_recv_per_rank=[33096, 32751, 32892, 32767, 32757, 32741, 32768, 32372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65186 expected_overflow=0 exact_recv_per_rank=[33096, 32751, 32892, 32767, 32757, 32741, 32768, 32372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65988 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[33096, 31471, 33095, 33096, 32099, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31471 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35513 max_load_reduction_pct=0.517607 moved_to_replicas=94372 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[36995, 30681, 24523, 26017, 5182, 27094, 43042, 68610] avg=32768.0000 max=68610 max_rank=7 min=5182 min_rank=4 imbalance_ratio=2.093811 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=15 calls=2 budget=40960 expected_recv=65186 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[36995, 30681, 24523, 26017, 5182, 27094, 43042, 68610] avg=32768.0000 max=68610 max_rank=7 min=5182 min_rank=4 imbalance_ratio=2.093811 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[36995, 30681, 24523, 26017, 5182, 27094, 43042, 68610] avg=32768.0000 max=68610 max_rank=7 min=5182 min_rank=4 imbalance_ratio=2.093811 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[36995, 30681, 24523, 26017, 5182, 27094, 43042, 68610] avg=32768.0000 max=68610 max_rank=7 min=5182 min_rank=4 imbalance_ratio=2.093811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[36995, 30681, 24523, 26017, 5182, 27094, 43042, 68610] avg=32768.0000 max=68610 max_rank=7 min=5182 min_rank=4 imbalance_ratio=2.093811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[33096, 31471, 33095, 33096, 32099, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31471 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35513 max_load_reduction_pct=0.517607 moved_to_replicas=94372 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[33096, 31471, 33095, 33096, 32099, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31471 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35513 max_load_reduction_pct=0.517607 moved_to_replicas=94372 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[4729, 32681, 130898, 29450, 32765, 1, 16, 31604] avg=32768.0000 max=130898 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.994690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[33096, 31471, 33095, 33096, 32099, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31471 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35513 max_load_reduction_pct=0.517607 moved_to_replicas=94372 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[33096, 31471, 33095, 33096, 32099, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31471 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35513 max_load_reduction_pct=0.517607 moved_to_replicas=94372 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[36995, 30681, 24523, 26017, 5182, 27094, 43042, 68610] avg=32768.0000 max=68610 max_rank=7 min=5182 min_rank=4 imbalance_ratio=2.093811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[36995, 30681, 24523, 26017, 5182, 27094, 43042, 68610] avg=32768.0000 max=68610 max_rank=7 min=5182 min_rank=4 imbalance_ratio=2.093811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[33096, 31471, 33095, 33096, 32099, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31471 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35513 max_load_reduction_pct=0.517607 moved_to_replicas=94372 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[33096, 32681, 33096, 33096, 32765, 32765, 32779, 31866] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97802 max_load_reduction_pct=0.747162 moved_to_replicas=195685 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[4729, 32681, 130898, 29450, 32765, 1, 16, 31604] avg=32768.0000 max=130898 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.994690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[4729, 32681, 130898, 29450, 32765, 1, 16, 31604] avg=32768.0000 max=130898 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.994690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[33096, 31471, 33095, 33096, 32099, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31471 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35513 max_load_reduction_pct=0.517607 moved_to_replicas=94372 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[4729, 32681, 130898, 29450, 32765, 1, 16, 31604] avg=32768.0000 max=130898 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.994690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[33096, 31471, 33095, 33096, 32099, 33095, 33095, 33097] avg=32768.0000 max=33097 max_rank=7 min=31471 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=35513 max_load_reduction_pct=0.517607 moved_to_replicas=94372 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[4729, 32681, 130898, 29450, 32765, 1, 16, 31604] avg=32768.0000 max=130898 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.994690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[4729, 32681, 130898, 29450, 32765, 1, 16, 31604] avg=32768.0000 max=130898 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.994690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[33096, 32681, 33096, 33096, 32765, 32765, 32779, 31866] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97802 max_load_reduction_pct=0.747162 moved_to_replicas=195685 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[32791, 3954, 88158, 32768, 3, 65319, 6304, 32847] avg=32768.0000 max=88158 max_rank=2 min=3 min_rank=4 imbalance_ratio=2.690369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[33096, 32681, 33096, 33096, 32765, 32765, 32779, 31866] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97802 max_load_reduction_pct=0.747162 moved_to_replicas=195685 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[33096, 32681, 33096, 33096, 32765, 32765, 32779, 31866] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97802 max_load_reduction_pct=0.747162 moved_to_replicas=195685 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[33096, 32681, 33096, 33096, 32765, 32765, 32779, 31866] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97802 max_load_reduction_pct=0.747162 moved_to_replicas=195685 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[4729, 32681, 130898, 29450, 32765, 1, 16, 31604] avg=32768.0000 max=130898 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.994690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[33096, 32681, 33096, 33096, 32765, 32765, 32779, 31866] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97802 max_load_reduction_pct=0.747162 moved_to_replicas=195685 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[4729, 32681, 130898, 29450, 32765, 1, 16, 31604] avg=32768.0000 max=130898 max_rank=2 min=1 min_rank=5 imbalance_ratio=3.994690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[32791, 31684, 33095, 32768, 32768, 33096, 33095, 32847] avg=32768.0000 max=33096 max_rank=5 min=31684 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=55062 max_load_reduction_pct=0.624583 moved_to_replicas=173362 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[32791, 3954, 88158, 32768, 3, 65319, 6304, 32847] avg=32768.0000 max=88158 max_rank=2 min=3 min_rank=4 imbalance_ratio=2.690369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[32791, 3954, 88158, 32768, 3, 65319, 6304, 32847] avg=32768.0000 max=88158 max_rank=2 min=3 min_rank=4 imbalance_ratio=2.690369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[32791, 3954, 88158, 32768, 3, 65319, 6304, 32847] avg=32768.0000 max=88158 max_rank=2 min=3 min_rank=4 imbalance_ratio=2.690369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[33096, 32681, 33096, 33096, 32765, 32765, 32779, 31866] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97802 max_load_reduction_pct=0.747162 moved_to_replicas=195685 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[33096, 32681, 33096, 33096, 32765, 32765, 32779, 31866] avg=32768.0000 max=33096 max_rank=0 min=31866 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=97802 max_load_reduction_pct=0.747162 moved_to_replicas=195685 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[32791, 3954, 88158, 32768, 3, 65319, 6304, 32847] avg=32768.0000 max=88158 max_rank=2 min=3 min_rank=4 imbalance_ratio=2.690369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[32791, 31684, 33095, 32768, 32768, 33096, 33095, 32847] avg=32768.0000 max=33096 max_rank=5 min=31684 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=55062 max_load_reduction_pct=0.624583 moved_to_replicas=173362 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[32791, 3954, 88158, 32768, 3, 65319, 6304, 32847] avg=32768.0000 max=88158 max_rank=2 min=3 min_rank=4 imbalance_ratio=2.690369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[2, 32664, 22075, 0, 75129, 32165, 34930, 65179] avg=32768.0000 max=75129 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.292755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[32791, 31684, 33095, 32768, 32768, 33096, 33095, 32847] avg=32768.0000 max=33096 max_rank=5 min=31684 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=55062 max_load_reduction_pct=0.624583 moved_to_replicas=173362 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[32791, 31684, 33095, 32768, 32768, 33096, 33095, 32847] avg=32768.0000 max=33096 max_rank=5 min=31684 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=55062 max_load_reduction_pct=0.624583 moved_to_replicas=173362 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[32791, 31684, 33095, 32768, 32768, 33096, 33095, 32847] avg=32768.0000 max=33096 max_rank=5 min=31684 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=55062 max_load_reduction_pct=0.624583 moved_to_replicas=173362 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[32791, 31684, 33095, 32768, 32768, 33096, 33095, 32847] avg=32768.0000 max=33096 max_rank=5 min=31684 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=55062 max_load_reduction_pct=0.624583 moved_to_replicas=173362 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[32791, 3954, 88158, 32768, 3, 65319, 6304, 32847] avg=32768.0000 max=88158 max_rank=2 min=3 min_rank=4 imbalance_ratio=2.690369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[33095, 32664, 32187, 32745, 33096, 32165, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32165 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42033 max_load_reduction_pct=0.559478 moved_to_replicas=150169 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[32791, 3954, 88158, 32768, 3, 65319, 6304, 32847] avg=32768.0000 max=88158 max_rank=2 min=3 min_rank=4 imbalance_ratio=2.690369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[2, 32664, 22075, 0, 75129, 32165, 34930, 65179] avg=32768.0000 max=75129 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.292755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[2, 32664, 22075, 0, 75129, 32165, 34930, 65179] avg=32768.0000 max=75129 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.292755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[2, 32664, 22075, 0, 75129, 32165, 34930, 65179] avg=32768.0000 max=75129 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.292755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[32791, 31684, 33095, 32768, 32768, 33096, 33095, 32847] avg=32768.0000 max=33096 max_rank=5 min=31684 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=55062 max_load_reduction_pct=0.624583 moved_to_replicas=173362 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[32791, 31684, 33095, 32768, 32768, 33096, 33095, 32847] avg=32768.0000 max=33096 max_rank=5 min=31684 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=55062 max_load_reduction_pct=0.624583 moved_to_replicas=173362 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[2, 32664, 22075, 0, 75129, 32165, 34930, 65179] avg=32768.0000 max=75129 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.292755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[33095, 32664, 32187, 32745, 33096, 32165, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32165 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42033 max_load_reduction_pct=0.559478 moved_to_replicas=150169 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[2, 32664, 22075, 0, 75129, 32165, 34930, 65179] avg=32768.0000 max=75129 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.292755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[30793, 66073, 65990, 32793, 736, 32762, 32768, 229] avg=32768.0000 max=66073 max_rank=1 min=229 min_rank=7 imbalance_ratio=2.016388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[33095, 32664, 32187, 32745, 33096, 32165, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32165 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42033 max_load_reduction_pct=0.559478 moved_to_replicas=150169 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[33095, 32664, 32187, 32745, 33096, 32165, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32165 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42033 max_load_reduction_pct=0.559478 moved_to_replicas=150169 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[33095, 32664, 32187, 32745, 33096, 32165, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32165 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42033 max_load_reduction_pct=0.559478 moved_to_replicas=150169 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[2, 32664, 22075, 0, 75129, 32165, 34930, 65179] avg=32768.0000 max=75129 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.292755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[33095, 32664, 32187, 32745, 33096, 32165, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32165 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42033 max_load_reduction_pct=0.559478 moved_to_replicas=150169 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[31538, 33096, 33096, 32793, 33096, 32762, 32768, 32995] avg=32768.0000 max=33096 max_rank=1 min=31538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32977 max_load_reduction_pct=0.499099 moved_to_replicas=132381 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[2, 32664, 22075, 0, 75129, 32165, 34930, 65179] avg=32768.0000 max=75129 max_rank=4 min=0 min_rank=3 imbalance_ratio=2.292755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[30793, 66073, 65990, 32793, 736, 32762, 32768, 229] avg=32768.0000 max=66073 max_rank=1 min=229 min_rank=7 imbalance_ratio=2.016388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[30793, 66073, 65990, 32793, 736, 32762, 32768, 229] avg=32768.0000 max=66073 max_rank=1 min=229 min_rank=7 imbalance_ratio=2.016388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[30793, 66073, 65990, 32793, 736, 32762, 32768, 229] avg=32768.0000 max=66073 max_rank=1 min=229 min_rank=7 imbalance_ratio=2.016388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[33095, 32664, 32187, 32745, 33096, 32165, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32165 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42033 max_load_reduction_pct=0.559478 moved_to_replicas=150169 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[31538, 33096, 33096, 32793, 33096, 32762, 32768, 32995] avg=32768.0000 max=33096 max_rank=1 min=31538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32977 max_load_reduction_pct=0.499099 moved_to_replicas=132381 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[30793, 66073, 65990, 32793, 736, 32762, 32768, 229] avg=32768.0000 max=66073 max_rank=1 min=229 min_rank=7 imbalance_ratio=2.016388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[33095, 32664, 32187, 32745, 33096, 32165, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=32165 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42033 max_load_reduction_pct=0.559478 moved_to_replicas=150169 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[65247, 91003, 3436, 32848, 632, 32768, 36209, 1] avg=32768.0000 max=91003 max_rank=1 min=1 min_rank=7 imbalance_ratio=2.777191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[30793, 66073, 65990, 32793, 736, 32762, 32768, 229] avg=32768.0000 max=66073 max_rank=1 min=229 min_rank=7 imbalance_ratio=2.016388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[31538, 33096, 33096, 32793, 33096, 32762, 32768, 32995] avg=32768.0000 max=33096 max_rank=1 min=31538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32977 max_load_reduction_pct=0.499099 moved_to_replicas=132381 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[31538, 33096, 33096, 32793, 33096, 32762, 32768, 32995] avg=32768.0000 max=33096 max_rank=1 min=31538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32977 max_load_reduction_pct=0.499099 moved_to_replicas=132381 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[31538, 33096, 33096, 32793, 33096, 32762, 32768, 32995] avg=32768.0000 max=33096 max_rank=1 min=31538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32977 max_load_reduction_pct=0.499099 moved_to_replicas=132381 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33095, 33096, 32848, 31378, 32768, 33096, 32767] avg=32768.0000 max=33096 max_rank=0 min=31378 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57907 max_load_reduction_pct=0.636320 moved_to_replicas=187384 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[31538, 33096, 33096, 32793, 33096, 32762, 32768, 32995] avg=32768.0000 max=33096 max_rank=1 min=31538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32977 max_load_reduction_pct=0.499099 moved_to_replicas=132381 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[30793, 66073, 65990, 32793, 736, 32762, 32768, 229] avg=32768.0000 max=66073 max_rank=1 min=229 min_rank=7 imbalance_ratio=2.016388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[65247, 91003, 3436, 32848, 632, 32768, 36209, 1] avg=32768.0000 max=91003 max_rank=1 min=1 min_rank=7 imbalance_ratio=2.777191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[30793, 66073, 65990, 32793, 736, 32762, 32768, 229] avg=32768.0000 max=66073 max_rank=1 min=229 min_rank=7 imbalance_ratio=2.016388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[65247, 91003, 3436, 32848, 632, 32768, 36209, 1] avg=32768.0000 max=91003 max_rank=1 min=1 min_rank=7 imbalance_ratio=2.777191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[65247, 91003, 3436, 32848, 632, 32768, 36209, 1] avg=32768.0000 max=91003 max_rank=1 min=1 min_rank=7 imbalance_ratio=2.777191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[31538, 33096, 33096, 32793, 33096, 32762, 32768, 32995] avg=32768.0000 max=33096 max_rank=1 min=31538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32977 max_load_reduction_pct=0.499099 moved_to_replicas=132381 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33095, 33096, 32848, 31378, 32768, 33096, 32767] avg=32768.0000 max=33096 max_rank=0 min=31378 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57907 max_load_reduction_pct=0.636320 moved_to_replicas=187384 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[65247, 91003, 3436, 32848, 632, 32768, 36209, 1] avg=32768.0000 max=91003 max_rank=1 min=1 min_rank=7 imbalance_ratio=2.777191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[31538, 33096, 33096, 32793, 33096, 32762, 32768, 32995] avg=32768.0000 max=33096 max_rank=1 min=31538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=32977 max_load_reduction_pct=0.499099 moved_to_replicas=132381 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32741, 2840, 62764, 65499, 32768, 2, 32762] avg=32768.0000 max=65499 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[65247, 91003, 3436, 32848, 632, 32768, 36209, 1] avg=32768.0000 max=91003 max_rank=1 min=1 min_rank=7 imbalance_ratio=2.777191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33095, 33096, 32848, 31378, 32768, 33096, 32767] avg=32768.0000 max=33096 max_rank=0 min=31378 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57907 max_load_reduction_pct=0.636320 moved_to_replicas=187384 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33095, 33096, 32848, 31378, 32768, 33096, 32767] avg=32768.0000 max=33096 max_rank=0 min=31378 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57907 max_load_reduction_pct=0.636320 moved_to_replicas=187384 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33095, 33096, 32848, 31378, 32768, 33096, 32767] avg=32768.0000 max=33096 max_rank=0 min=31378 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57907 max_load_reduction_pct=0.636320 moved_to_replicas=187384 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32741, 32508, 33096, 33096, 32768, 32405, 32762] avg=32768.0000 max=33096 max_rank=3 min=32405 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32403 max_load_reduction_pct=0.494710 moved_to_replicas=123762 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33095, 33096, 32848, 31378, 32768, 33096, 32767] avg=32768.0000 max=33096 max_rank=0 min=31378 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57907 max_load_reduction_pct=0.636320 moved_to_replicas=187384 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32741, 2840, 62764, 65499, 32768, 2, 32762] avg=32768.0000 max=65499 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[65247, 91003, 3436, 32848, 632, 32768, 36209, 1] avg=32768.0000 max=91003 max_rank=1 min=1 min_rank=7 imbalance_ratio=2.777191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[65247, 91003, 3436, 32848, 632, 32768, 36209, 1] avg=32768.0000 max=91003 max_rank=1 min=1 min_rank=7 imbalance_ratio=2.777191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32741, 2840, 62764, 65499, 32768, 2, 32762] avg=32768.0000 max=65499 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32741, 2840, 62764, 65499, 32768, 2, 32762] avg=32768.0000 max=65499 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32741, 32508, 33096, 33096, 32768, 32405, 32762] avg=32768.0000 max=33096 max_rank=3 min=32405 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32403 max_load_reduction_pct=0.494710 moved_to_replicas=123762 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33095, 33096, 32848, 31378, 32768, 33096, 32767] avg=32768.0000 max=33096 max_rank=0 min=31378 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57907 max_load_reduction_pct=0.636320 moved_to_replicas=187384 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32741, 2840, 62764, 65499, 32768, 2, 32762] avg=32768.0000 max=65499 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33095, 33096, 32848, 31378, 32768, 33096, 32767] avg=32768.0000 max=33096 max_rank=0 min=31378 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=57907 max_load_reduction_pct=0.636320 moved_to_replicas=187384 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[65497, 35, 65477, 83, 65535, 32760, 32752, 5] avg=32768.0000 max=65535 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32741, 2840, 62764, 65499, 32768, 2, 32762] avg=32768.0000 max=65499 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32741, 32508, 33096, 33096, 32768, 32405, 32762] avg=32768.0000 max=33096 max_rank=3 min=32405 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32403 max_load_reduction_pct=0.494710 moved_to_replicas=123762 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32741, 32508, 33096, 33096, 32768, 32405, 32762] avg=32768.0000 max=33096 max_rank=3 min=32405 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32403 max_load_reduction_pct=0.494710 moved_to_replicas=123762 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32741, 32508, 33096, 33096, 32768, 32405, 32762] avg=32768.0000 max=33096 max_rank=3 min=32405 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32403 max_load_reduction_pct=0.494710 moved_to_replicas=123762 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32436, 33096, 32464, 33096, 32760, 32752, 32444] avg=32768.0000 max=33096 max_rank=0 min=32436 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=194381 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32741, 32508, 33096, 33096, 32768, 32405, 32762] avg=32768.0000 max=33096 max_rank=3 min=32405 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32403 max_load_reduction_pct=0.494710 moved_to_replicas=123762 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[65497, 35, 65477, 83, 65535, 32760, 32752, 5] avg=32768.0000 max=65535 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32741, 2840, 62764, 65499, 32768, 2, 32762] avg=32768.0000 max=65499 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32741, 2840, 62764, 65499, 32768, 2, 32762] avg=32768.0000 max=65499 max_rank=4 min=2 min_rank=6 imbalance_ratio=1.998871 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[65497, 35, 65477, 83, 65535, 32760, 32752, 5] avg=32768.0000 max=65535 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[65497, 35, 65477, 83, 65535, 32760, 32752, 5] avg=32768.0000 max=65535 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32436, 33096, 32464, 33096, 32760, 32752, 32444] avg=32768.0000 max=33096 max_rank=0 min=32436 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=194381 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32741, 32508, 33096, 33096, 32768, 32405, 32762] avg=32768.0000 max=33096 max_rank=3 min=32405 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32403 max_load_reduction_pct=0.494710 moved_to_replicas=123762 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[65497, 35, 65477, 83, 65535, 32760, 32752, 5] avg=32768.0000 max=65535 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[14233, 31402, 98301, 22902, 2640, 32769, 2, 59895] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[32768, 32741, 32508, 33096, 33096, 32768, 32405, 32762] avg=32768.0000 max=33096 max_rank=3 min=32405 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32403 max_load_reduction_pct=0.494710 moved_to_replicas=123762 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[65497, 35, 65477, 83, 65535, 32760, 32752, 5] avg=32768.0000 max=65535 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32436, 33096, 32464, 33096, 32760, 32752, 32444] avg=32768.0000 max=33096 max_rank=0 min=32436 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=194381 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32436, 33096, 32464, 33096, 32760, 32752, 32444] avg=32768.0000 max=33096 max_rank=0 min=32436 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=194381 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32436, 33096, 32464, 33096, 32760, 32752, 32444] avg=32768.0000 max=33096 max_rank=0 min=32436 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=194381 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[33095, 31402, 33096, 32823, 33096, 32769, 32768, 33095] avg=32768.0000 max=33096 max_rank=2 min=31402 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=180726 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[14233, 31402, 98301, 22902, 2640, 32769, 2, 59895] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32436, 33096, 32464, 33096, 32760, 32752, 32444] avg=32768.0000 max=33096 max_rank=0 min=32436 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=194381 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[65497, 35, 65477, 83, 65535, 32760, 32752, 5] avg=32768.0000 max=65535 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[65497, 35, 65477, 83, 65535, 32760, 32752, 5] avg=32768.0000 max=65535 max_rank=4 min=5 min_rank=7 imbalance_ratio=1.999969 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[14233, 31402, 98301, 22902, 2640, 32769, 2, 59895] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[14233, 31402, 98301, 22902, 2640, 32769, 2, 59895] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[33095, 31402, 33096, 32823, 33096, 32769, 32768, 33095] avg=32768.0000 max=33096 max_rank=2 min=31402 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=180726 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32436, 33096, 32464, 33096, 32760, 32752, 32444] avg=32768.0000 max=33096 max_rank=0 min=32436 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=194381 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[9, 4, 32028, 1601, 97160, 33068, 25, 98249] avg=32768.0000 max=98249 max_rank=7 min=4 min_rank=1 imbalance_ratio=2.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[14233, 31402, 98301, 22902, 2640, 32769, 2, 59895] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[33096, 32436, 33096, 32464, 33096, 32760, 32752, 32444] avg=32768.0000 max=33096 max_rank=0 min=32436 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32439 max_load_reduction_pct=0.494987 moved_to_replicas=194381 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[14233, 31402, 98301, 22902, 2640, 32769, 2, 59895] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[33095, 31402, 33096, 32823, 33096, 32769, 32768, 33095] avg=32768.0000 max=33096 max_rank=2 min=31402 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=180726 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[33095, 31402, 33096, 32823, 33096, 32769, 32768, 33095] avg=32768.0000 max=33096 max_rank=2 min=31402 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=180726 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[32398, 32768, 32028, 32899, 33096, 33068, 32791, 33096] avg=32768.0000 max=33096 max_rank=4 min=32028 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65153 max_load_reduction_pct=0.663142 moved_to_replicas=258594 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[9, 4, 32028, 1601, 97160, 33068, 25, 98249] avg=32768.0000 max=98249 max_rank=7 min=4 min_rank=1 imbalance_ratio=2.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[33095, 31402, 33096, 32823, 33096, 32769, 32768, 33095] avg=32768.0000 max=33096 max_rank=2 min=31402 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=180726 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[33095, 31402, 33096, 32823, 33096, 32769, 32768, 33095] avg=32768.0000 max=33096 max_rank=2 min=31402 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=180726 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[14233, 31402, 98301, 22902, 2640, 32769, 2, 59895] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[14233, 31402, 98301, 22902, 2640, 32769, 2, 59895] avg=32768.0000 max=98301 max_rank=2 min=2 min_rank=6 imbalance_ratio=2.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[9, 4, 32028, 1601, 97160, 33068, 25, 98249] avg=32768.0000 max=98249 max_rank=7 min=4 min_rank=1 imbalance_ratio=2.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[32398, 32768, 32028, 32899, 33096, 33068, 32791, 33096] avg=32768.0000 max=33096 max_rank=4 min=32028 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65153 max_load_reduction_pct=0.663142 moved_to_replicas=258594 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[9, 4, 32028, 1601, 97160, 33068, 25, 98249] avg=32768.0000 max=98249 max_rank=7 min=4 min_rank=1 imbalance_ratio=2.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[65331, 2, 2, 34468, 65536, 239, 63805, 32761] avg=32768.0000 max=65536 max_rank=4 min=2 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[33095, 31402, 33096, 32823, 33096, 32769, 32768, 33095] avg=32768.0000 max=33096 max_rank=2 min=31402 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=180726 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[33095, 31402, 33096, 32823, 33096, 32769, 32768, 33095] avg=32768.0000 max=33096 max_rank=2 min=31402 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65205 max_load_reduction_pct=0.663320 moved_to_replicas=180726 active_clone_links=11 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[32398, 32768, 32028, 32899, 33096, 33068, 32791, 33096] avg=32768.0000 max=33096 max_rank=4 min=32028 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65153 max_load_reduction_pct=0.663142 moved_to_replicas=258594 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[9, 4, 32028, 1601, 97160, 33068, 25, 98249] avg=32768.0000 max=98249 max_rank=7 min=4 min_rank=1 imbalance_ratio=2.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[9, 4, 32028, 1601, 97160, 33068, 25, 98249] avg=32768.0000 max=98249 max_rank=7 min=4 min_rank=1 imbalance_ratio=2.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[32398, 32768, 32028, 32899, 33096, 33068, 32791, 33096] avg=32768.0000 max=33096 max_rank=4 min=32028 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65153 max_load_reduction_pct=0.663142 moved_to_replicas=258594 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32442, 32237, 33096, 33096, 32320, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=32237 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=192762 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[65331, 2, 2, 34468, 65536, 239, 63805, 32761] avg=32768.0000 max=65536 max_rank=4 min=2 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[32398, 32768, 32028, 32899, 33096, 33068, 32791, 33096] avg=32768.0000 max=33096 max_rank=4 min=32028 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65153 max_load_reduction_pct=0.663142 moved_to_replicas=258594 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[9, 4, 32028, 1601, 97160, 33068, 25, 98249] avg=32768.0000 max=98249 max_rank=7 min=4 min_rank=1 imbalance_ratio=2.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[32398, 32768, 32028, 32899, 33096, 33068, 32791, 33096] avg=32768.0000 max=33096 max_rank=4 min=32028 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65153 max_load_reduction_pct=0.663142 moved_to_replicas=258594 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[9, 4, 32028, 1601, 97160, 33068, 25, 98249] avg=32768.0000 max=98249 max_rank=7 min=4 min_rank=1 imbalance_ratio=2.998322 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[65331, 2, 2, 34468, 65536, 239, 63805, 32761] avg=32768.0000 max=65536 max_rank=4 min=2 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32442, 32237, 33096, 33096, 32320, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=32237 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=192762 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[65331, 2, 2, 34468, 65536, 239, 63805, 32761] avg=32768.0000 max=65536 max_rank=4 min=2 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[32723, 65531, 33187, 98028, 32622, 2, 3, 48] avg=32768.0000 max=98028 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.991577 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[32398, 32768, 32028, 32899, 33096, 33068, 32791, 33096] avg=32768.0000 max=33096 max_rank=4 min=32028 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65153 max_load_reduction_pct=0.663142 moved_to_replicas=258594 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[32398, 32768, 32028, 32899, 33096, 33068, 32791, 33096] avg=32768.0000 max=33096 max_rank=4 min=32028 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=65153 max_load_reduction_pct=0.663142 moved_to_replicas=258594 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32442, 32237, 33096, 33096, 32320, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=32237 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=192762 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[65331, 2, 2, 34468, 65536, 239, 63805, 32761] avg=32768.0000 max=65536 max_rank=4 min=2 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[65331, 2, 2, 34468, 65536, 239, 63805, 32761] avg=32768.0000 max=65536 max_rank=4 min=2 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32442, 32237, 33096, 33096, 32320, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=32237 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=192762 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[32723, 33096, 33096, 33096, 32622, 32768, 32260, 32483] avg=32768.0000 max=33096 max_rank=1 min=32260 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64932 max_load_reduction_pct=0.662382 moved_to_replicas=194959 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[32723, 65531, 33187, 98028, 32622, 2, 3, 48] avg=32768.0000 max=98028 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.991577 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32442, 32237, 33096, 33096, 32320, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=32237 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=192762 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[65331, 2, 2, 34468, 65536, 239, 63805, 32761] avg=32768.0000 max=65536 max_rank=4 min=2 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32442, 32237, 33096, 33096, 32320, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=32237 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=192762 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[32723, 65531, 33187, 98028, 32622, 2, 3, 48] avg=32768.0000 max=98028 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.991577 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[65331, 2, 2, 34468, 65536, 239, 63805, 32761] avg=32768.0000 max=65536 max_rank=4 min=2 min_rank=1 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[32723, 33096, 33096, 33096, 32622, 32768, 32260, 32483] avg=32768.0000 max=33096 max_rank=1 min=32260 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64932 max_load_reduction_pct=0.662382 moved_to_replicas=194959 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[5, 32745, 75678, 32771, 32767, 22413, 33008, 32757] avg=32768.0000 max=75678 max_rank=2 min=5 min_rank=0 imbalance_ratio=2.309509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[32723, 65531, 33187, 98028, 32622, 2, 3, 48] avg=32768.0000 max=98028 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.991577 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32442, 32237, 33096, 33096, 32320, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=32237 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=192762 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[32723, 33096, 33096, 33096, 32622, 32768, 32260, 32483] avg=32768.0000 max=33096 max_rank=1 min=32260 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64932 max_load_reduction_pct=0.662382 moved_to_replicas=194959 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[33096, 32442, 32237, 33096, 33096, 32320, 33096, 32761] avg=32768.0000 max=33096 max_rank=0 min=32237 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32440 max_load_reduction_pct=0.494995 moved_to_replicas=192762 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[32723, 65531, 33187, 98028, 32622, 2, 3, 48] avg=32768.0000 max=98028 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.991577 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[32723, 65531, 33187, 98028, 32622, 2, 3, 48] avg=32768.0000 max=98028 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.991577 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[5, 32745, 75678, 32771, 32767, 22413, 33008, 32757] avg=32768.0000 max=75678 max_rank=2 min=5 min_rank=0 imbalance_ratio=2.309509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[32771, 32745, 33096, 32771, 32767, 32229, 33008, 32757] avg=32768.0000 max=33096 max_rank=2 min=32229 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42582 max_load_reduction_pct=0.562673 moved_to_replicas=86621 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[32723, 33096, 33096, 33096, 32622, 32768, 32260, 32483] avg=32768.0000 max=33096 max_rank=1 min=32260 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64932 max_load_reduction_pct=0.662382 moved_to_replicas=194959 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[32723, 33096, 33096, 33096, 32622, 32768, 32260, 32483] avg=32768.0000 max=33096 max_rank=1 min=32260 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64932 max_load_reduction_pct=0.662382 moved_to_replicas=194959 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[32723, 33096, 33096, 33096, 32622, 32768, 32260, 32483] avg=32768.0000 max=33096 max_rank=1 min=32260 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64932 max_load_reduction_pct=0.662382 moved_to_replicas=194959 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[32723, 65531, 33187, 98028, 32622, 2, 3, 48] avg=32768.0000 max=98028 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.991577 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[32771, 32745, 33096, 32771, 32767, 32229, 33008, 32757] avg=32768.0000 max=33096 max_rank=2 min=32229 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42582 max_load_reduction_pct=0.562673 moved_to_replicas=86621 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[5, 32745, 75678, 32771, 32767, 22413, 33008, 32757] avg=32768.0000 max=75678 max_rank=2 min=5 min_rank=0 imbalance_ratio=2.309509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[32723, 65531, 33187, 98028, 32622, 2, 3, 48] avg=32768.0000 max=98028 max_rank=3 min=2 min_rank=5 imbalance_ratio=2.991577 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[31207, 40, 64411, 2, 65530, 32769, 32767, 35418] avg=32768.0000 max=65530 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[5, 32745, 75678, 32771, 32767, 22413, 33008, 32757] avg=32768.0000 max=75678 max_rank=2 min=5 min_rank=0 imbalance_ratio=2.309509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[32723, 33096, 33096, 33096, 32622, 32768, 32260, 32483] avg=32768.0000 max=33096 max_rank=1 min=32260 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64932 max_load_reduction_pct=0.662382 moved_to_replicas=194959 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[5, 32745, 75678, 32771, 32767, 22413, 33008, 32757] avg=32768.0000 max=75678 max_rank=2 min=5 min_rank=0 imbalance_ratio=2.309509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[32771, 32745, 33096, 32771, 32767, 32229, 33008, 32757] avg=32768.0000 max=33096 max_rank=2 min=32229 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42582 max_load_reduction_pct=0.562673 moved_to_replicas=86621 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[32723, 33096, 33096, 33096, 32622, 32768, 32260, 32483] avg=32768.0000 max=33096 max_rank=1 min=32260 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=64932 max_load_reduction_pct=0.662382 moved_to_replicas=194959 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[5, 32745, 75678, 32771, 32767, 22413, 33008, 32757] avg=32768.0000 max=75678 max_rank=2 min=5 min_rank=0 imbalance_ratio=2.309509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[33096, 31788, 33096, 32436, 33096, 32769, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=133629 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[31207, 40, 64411, 2, 65530, 32769, 32767, 35418] avg=32768.0000 max=65530 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[32771, 32745, 33096, 32771, 32767, 32229, 33008, 32757] avg=32768.0000 max=33096 max_rank=2 min=32229 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42582 max_load_reduction_pct=0.562673 moved_to_replicas=86621 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[32771, 32745, 33096, 32771, 32767, 32229, 33008, 32757] avg=32768.0000 max=33096 max_rank=2 min=32229 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42582 max_load_reduction_pct=0.562673 moved_to_replicas=86621 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[32771, 32745, 33096, 32771, 32767, 32229, 33008, 32757] avg=32768.0000 max=33096 max_rank=2 min=32229 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42582 max_load_reduction_pct=0.562673 moved_to_replicas=86621 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[33096, 31788, 33096, 32436, 33096, 32769, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=133629 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[5, 32745, 75678, 32771, 32767, 22413, 33008, 32757] avg=32768.0000 max=75678 max_rank=2 min=5 min_rank=0 imbalance_ratio=2.309509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[31207, 40, 64411, 2, 65530, 32769, 32767, 35418] avg=32768.0000 max=65530 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[5, 32745, 75678, 32771, 32767, 22413, 33008, 32757] avg=32768.0000 max=75678 max_rank=2 min=5 min_rank=0 imbalance_ratio=2.309509 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[1, 65310, 4, 32766, 65727, 32769, 186, 65381] avg=32768.0000 max=65727 max_rank=4 min=1 min_rank=0 imbalance_ratio=2.005829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[31207, 40, 64411, 2, 65530, 32769, 32767, 35418] avg=32768.0000 max=65530 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[32771, 32745, 33096, 32771, 32767, 32229, 33008, 32757] avg=32768.0000 max=33096 max_rank=2 min=32229 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42582 max_load_reduction_pct=0.562673 moved_to_replicas=86621 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[33096, 31788, 33096, 32436, 33096, 32769, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=133629 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[31207, 40, 64411, 2, 65530, 32769, 32767, 35418] avg=32768.0000 max=65530 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[32771, 32745, 33096, 32771, 32767, 32229, 33008, 32757] avg=32768.0000 max=33096 max_rank=2 min=32229 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42582 max_load_reduction_pct=0.562673 moved_to_replicas=86621 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[31207, 40, 64411, 2, 65530, 32769, 32767, 35418] avg=32768.0000 max=65530 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[1, 65310, 4, 32766, 65727, 32769, 186, 65381] avg=32768.0000 max=65727 max_rank=4 min=1 min_rank=0 imbalance_ratio=2.005829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[32632, 33096, 32289, 32766, 33096, 32769, 32400, 33096] avg=32768.0000 max=33096 max_rank=1 min=32289 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32631 max_load_reduction_pct=0.496463 moved_to_replicas=194226 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[33096, 31788, 33096, 32436, 33096, 32769, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=133629 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[33096, 31788, 33096, 32436, 33096, 32769, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=133629 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[33096, 31788, 33096, 32436, 33096, 32769, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=133629 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[32632, 33096, 32289, 32766, 33096, 32769, 32400, 33096] avg=32768.0000 max=33096 max_rank=1 min=32289 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32631 max_load_reduction_pct=0.496463 moved_to_replicas=194226 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[31207, 40, 64411, 2, 65530, 32769, 32767, 35418] avg=32768.0000 max=65530 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[1, 65310, 4, 32766, 65727, 32769, 186, 65381] avg=32768.0000 max=65727 max_rank=4 min=1 min_rank=0 imbalance_ratio=2.005829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[31207, 40, 64411, 2, 65530, 32769, 32767, 35418] avg=32768.0000 max=65530 max_rank=4 min=2 min_rank=3 imbalance_ratio=1.999817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[98223, 32751, 32892, 32767, 1, 32741, 32768, 1] avg=32768.0000 max=98223 max_rank=0 min=1 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[32632, 33096, 32289, 32766, 33096, 32769, 32400, 33096] avg=32768.0000 max=33096 max_rank=1 min=32289 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32631 max_load_reduction_pct=0.496463 moved_to_replicas=194226 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[33096, 31788, 33096, 32436, 33096, 32769, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=133629 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[1, 65310, 4, 32766, 65727, 32769, 186, 65381] avg=32768.0000 max=65727 max_rank=4 min=1 min_rank=0 imbalance_ratio=2.005829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[1, 65310, 4, 32766, 65727, 32769, 186, 65381] avg=32768.0000 max=65727 max_rank=4 min=1 min_rank=0 imbalance_ratio=2.005829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[33096, 31788, 33096, 32436, 33096, 32769, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31788 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32434 max_load_reduction_pct=0.494949 moved_to_replicas=133629 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[33096, 32751, 32892, 32767, 32757, 32741, 32768, 32372] avg=32768.0000 max=33096 max_rank=0 min=32372 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=130285 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[98223, 32751, 32892, 32767, 1, 32741, 32768, 1] avg=32768.0000 max=98223 max_rank=0 min=1 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[1, 65310, 4, 32766, 65727, 32769, 186, 65381] avg=32768.0000 max=65727 max_rank=4 min=1 min_rank=0 imbalance_ratio=2.005829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[32632, 33096, 32289, 32766, 33096, 32769, 32400, 33096] avg=32768.0000 max=33096 max_rank=1 min=32289 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32631 max_load_reduction_pct=0.496463 moved_to_replicas=194226 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[32632, 33096, 32289, 32766, 33096, 32769, 32400, 33096] avg=32768.0000 max=33096 max_rank=1 min=32289 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32631 max_load_reduction_pct=0.496463 moved_to_replicas=194226 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[33096, 32751, 32892, 32767, 32757, 32741, 32768, 32372] avg=32768.0000 max=33096 max_rank=0 min=32372 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=130285 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[32632, 33096, 32289, 32766, 33096, 32769, 32400, 33096] avg=32768.0000 max=33096 max_rank=1 min=32289 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32631 max_load_reduction_pct=0.496463 moved_to_replicas=194226 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[98223, 32751, 32892, 32767, 1, 32741, 32768, 1] avg=32768.0000 max=98223 max_rank=0 min=1 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[1, 65310, 4, 32766, 65727, 32769, 186, 65381] avg=32768.0000 max=65727 max_rank=4 min=1 min_rank=0 imbalance_ratio=2.005829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[1, 65310, 4, 32766, 65727, 32769, 186, 65381] avg=32768.0000 max=65727 max_rank=4 min=1 min_rank=0 imbalance_ratio=2.005829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[33096, 32751, 32892, 32767, 32757, 32741, 32768, 32372] avg=32768.0000 max=33096 max_rank=0 min=32372 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=130285 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[32632, 33096, 32289, 32766, 33096, 32769, 32400, 33096] avg=32768.0000 max=33096 max_rank=1 min=32289 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32631 max_load_reduction_pct=0.496463 moved_to_replicas=194226 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[98223, 32751, 32892, 32767, 1, 32741, 32768, 1] avg=32768.0000 max=98223 max_rank=0 min=1 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[98223, 32751, 32892, 32767, 1, 32741, 32768, 1] avg=32768.0000 max=98223 max_rank=0 min=1 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[32632, 33096, 32289, 32766, 33096, 32769, 32400, 33096] avg=32768.0000 max=33096 max_rank=1 min=32289 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32631 max_load_reduction_pct=0.496463 moved_to_replicas=194226 active_clone_links=7 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[98223, 32751, 32892, 32767, 1, 32741, 32768, 1] avg=32768.0000 max=98223 max_rank=0 min=1 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[33096, 32751, 32892, 32767, 32757, 32741, 32768, 32372] avg=32768.0000 max=33096 max_rank=0 min=32372 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=130285 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[33096, 32751, 32892, 32767, 32757, 32741, 32768, 32372] avg=32768.0000 max=33096 max_rank=0 min=32372 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=130285 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[33096, 32751, 32892, 32767, 32757, 32741, 32768, 32372] avg=32768.0000 max=33096 max_rank=0 min=32372 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=130285 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[98223, 32751, 32892, 32767, 1, 32741, 32768, 1] avg=32768.0000 max=98223 max_rank=0 min=1 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[98223, 32751, 32892, 32767, 1, 32741, 32768, 1] avg=32768.0000 max=98223 max_rank=0 min=1 min_rank=4 imbalance_ratio=2.997528 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[33096, 32751, 32892, 32767, 32757, 32741, 32768, 32372] avg=32768.0000 max=33096 max_rank=0 min=32372 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=130285 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[33096, 32751, 32892, 32767, 32757, 32741, 32768, 32372] avg=32768.0000 max=33096 max_rank=0 min=32372 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=65127 max_load_reduction_pct=0.663052 moved_to_replicas=130285 active_clone_links=5 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=32 total_expected_recv=1045790 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=256512 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=32 total_expected_recv=1040141 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=259328 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=32 total_expected_recv=1051018 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=251520 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=32 total_expected_recv=1051564 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=248448 peak_demand=33099 required_capacity=1.010101 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=32 total_expected_recv=1049344 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=252032 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=32 total_expected_recv=1050232 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=252416 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=32 total_expected_recv=1049806 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=249216 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=32 total_expected_recv=1050713 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=249088 peak_demand=33096 required_capacity=1.010010 + [2026-08-30 05:10:31] iteration 11/ 12 | consumed samples: 176 | elapsed time per iteration (ms): 584.6 | throughput per GPU (TFLOP/s/GPU): 570.3 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.975285E+00 | loss scale: 1.0 | grad norm: 5.222 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32908, 33096, 30684, 33096, 33074, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32908, 33096, 30684, 33096, 33074, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32908, 33096, 30684, 33096, 33074, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=40960 expected_recv=65420 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32908, 33096, 30684, 33096, 33074, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32908, 33096, 30684, 33096, 33074, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=40960 expected_recv=66068 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32908, 33096, 30684, 33096, 33074, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32908, 33096, 30684, 33096, 33074, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=2 budget=40960 expected_recv=65420 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=40960 expected_recv=61842 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32908, 33096, 30684, 33096, 33074, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=2 budget=40960 expected_recv=66068 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=2 budget=40960 expected_recv=61842 actual_valid_recv=63232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=2 received_num_tokens=81920 valid_num_tokens=63232 slack_slots=18688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65523 expected_overflow=0 exact_recv_per_rank=[33095, 31396, 33096, 33096, 32765, 32760, 32853, 33083] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33095, 31396, 33096, 33096, 32765, 32760, 32853, 33083] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=40960 expected_recv=63214 expected_overflow=0 exact_recv_per_rank=[33095, 31396, 33096, 33096, 32765, 32760, 32853, 33083] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65523 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 31396, 33096, 33096, 32765, 32760, 32853, 33083] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=2 budget=40960 expected_recv=63214 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33095, 31396, 33096, 33096, 32765, 32760, 32853, 33083] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65719 expected_overflow=0 exact_recv_per_rank=[33095, 31396, 33096, 33096, 32765, 32760, 32853, 33083] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65732 expected_overflow=0 exact_recv_per_rank=[33095, 31396, 33096, 33096, 32765, 32760, 32853, 33083] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65719 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65732 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65525 expected_overflow=0 exact_recv_per_rank=[33095, 31396, 33096, 33096, 32765, 32760, 32853, 33083] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=2 budget=40960 expected_recv=65525 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32782, 33096, 32762, 32263, 31953, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=40960 expected_recv=63882 expected_overflow=0 exact_recv_per_rank=[33096, 32782, 33096, 32762, 32263, 31953, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=40960 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[33096, 32782, 33096, 32762, 32263, 31953, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=2 budget=40960 expected_recv=63882 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=2 budget=40960 expected_recv=65550 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 32782, 33096, 32762, 32263, 31953, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=40960 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[33096, 32782, 33096, 32762, 32263, 31953, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 32782, 33096, 32762, 32263, 31953, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32782, 33096, 32762, 32263, 31953, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=2 budget=40960 expected_recv=65531 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=40960 expected_recv=64559 expected_overflow=0 exact_recv_per_rank=[33096, 32782, 33096, 32762, 32263, 31953, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=2 budget=40960 expected_recv=64559 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31294, 32871, 32856, 32738, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=40960 expected_recv=63095 expected_overflow=0 exact_recv_per_rank=[31294, 32871, 32856, 32738, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=2 budget=40960 expected_recv=63095 actual_valid_recv=64000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65636 expected_overflow=0 exact_recv_per_rank=[31294, 32871, 32856, 32738, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=64000 slack_slots=17920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65636 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65290 expected_overflow=0 exact_recv_per_rank=[31294, 32871, 32856, 32738, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65498 expected_overflow=0 exact_recv_per_rank=[31294, 32871, 32856, 32738, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[31294, 32871, 32856, 32738, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31294, 32871, 32856, 32738, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65290 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=2 budget=40960 expected_recv=65498 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31294, 32871, 32856, 32738, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=40960 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[32363, 33096, 33096, 33097, 32196, 33095, 32762, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=40960 expected_recv=64642 expected_overflow=0 exact_recv_per_rank=[32363, 33096, 33096, 33097, 32196, 33095, 32762, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=2 budget=40960 expected_recv=66190 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32363, 33096, 33096, 33097, 32196, 33095, 32762, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=2 budget=40960 expected_recv=64642 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32363, 33096, 33096, 33097, 32196, 33095, 32762, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=40960 expected_recv=65525 expected_overflow=0 exact_recv_per_rank=[32363, 33096, 33096, 33097, 32196, 33095, 32762, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[32363, 33096, 33096, 33097, 32196, 33095, 32762, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=40960 expected_recv=64642 expected_overflow=0 exact_recv_per_rank=[32363, 33096, 33096, 33097, 32196, 33095, 32762, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=2 budget=40960 expected_recv=65525 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=2 budget=40960 expected_recv=64642 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=40960 expected_recv=64710 expected_overflow=0 exact_recv_per_rank=[32363, 33096, 33096, 33097, 32196, 33095, 32762, 32439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=40960 expected_recv=65537 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32212, 33097, 32019, 32766, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=2 budget=40960 expected_recv=64710 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32212, 33097, 32019, 32766, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=2 budget=40960 expected_recv=65537 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32212, 33097, 32019, 32766, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=40960 expected_recv=64504 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32212, 33097, 32019, 32766, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=40960 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32212, 33097, 32019, 32766, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=2 budget=40960 expected_recv=64504 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=40960 expected_recv=66194 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32212, 33097, 32019, 32766, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=40960 expected_recv=65504 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32212, 33097, 32019, 32766, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=2 budget=40960 expected_recv=66190 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=2 budget=40960 expected_recv=66194 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=40960 expected_recv=63974 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32212, 33097, 32019, 32766, 33096, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=40960 expected_recv=66097 expected_overflow=0 exact_recv_per_rank=[32763, 33094, 31564, 33096, 33096, 33001, 32770, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=2 budget=40960 expected_recv=65504 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=2 budget=40960 expected_recv=63974 actual_valid_recv=65280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65521 expected_overflow=0 exact_recv_per_rank=[32763, 33094, 31564, 33096, 33096, 33001, 32770, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=2 budget=40960 expected_recv=66097 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=2 received_num_tokens=81920 valid_num_tokens=65280 slack_slots=16640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=40960 expected_recv=66189 expected_overflow=0 exact_recv_per_rank=[32763, 33094, 31564, 33096, 33096, 33001, 32770, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65521 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=2 budget=40960 expected_recv=66189 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=40960 expected_recv=63021 expected_overflow=0 exact_recv_per_rank=[32763, 33094, 31564, 33096, 33096, 33001, 32770, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=2 budget=40960 expected_recv=63021 actual_valid_recv=64256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[32763, 33094, 31564, 33096, 33096, 33001, 32770, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65553 expected_overflow=0 exact_recv_per_rank=[32763, 33094, 31564, 33096, 33096, 33001, 32770, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65522 expected_overflow=0 exact_recv_per_rank=[32763, 33094, 31564, 33096, 33096, 33001, 32770, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=64256 slack_slots=17664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32763, 33094, 31564, 33096, 33096, 33001, 32770, 32760] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65553 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=40960 expected_recv=64887 expected_overflow=0 exact_recv_per_rank=[33098, 32114, 33096, 33094, 33096, 32766, 32430, 32450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=2 budget=40960 expected_recv=65522 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=2 budget=40960 expected_recv=64887 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=40960 expected_recv=66193 expected_overflow=0 exact_recv_per_rank=[33098, 32114, 33096, 33094, 33096, 32766, 32430, 32450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=40960 expected_recv=64920 expected_overflow=0 exact_recv_per_rank=[33098, 32114, 33096, 33094, 33096, 32766, 32430, 32450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=2 budget=40960 expected_recv=66193 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33098, 32114, 33096, 33094, 33096, 32766, 32430, 32450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=2 budget=40960 expected_recv=64920 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=40960 expected_recv=64815 expected_overflow=0 exact_recv_per_rank=[33098, 32114, 33096, 33094, 33096, 32766, 32430, 32450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33098, 32114, 33096, 33094, 33096, 32766, 32430, 32450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=40960 expected_recv=64898 expected_overflow=0 exact_recv_per_rank=[33098, 32114, 33096, 33094, 33096, 32766, 32430, 32450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33098, 32114, 33096, 33094, 33096, 32766, 32430, 32450] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=40960 expected_recv=64829 expected_overflow=0 exact_recv_per_rank=[31558, 32732, 33096, 32694, 33096, 33096, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=2 budget=40960 expected_recv=64815 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=2 budget=40960 expected_recv=64898 actual_valid_recv=65536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=2 budget=40960 expected_recv=64829 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=65536 slack_slots=16384 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=40960 expected_recv=64454 expected_overflow=0 exact_recv_per_rank=[31558, 32732, 33096, 32694, 33096, 33096, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65433 expected_overflow=0 exact_recv_per_rank=[31558, 32732, 33096, 32694, 33096, 33096, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=2 budget=40960 expected_recv=64454 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65433 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31558, 32732, 33096, 32694, 33096, 33096, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65591 expected_overflow=0 exact_recv_per_rank=[31558, 32732, 33096, 32694, 33096, 33096, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[31558, 32732, 33096, 32694, 33096, 33096, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65406 expected_overflow=0 exact_recv_per_rank=[31558, 32732, 33096, 32694, 33096, 33096, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32799, 30771, 33096, 33095, 33095, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65591 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[31558, 32732, 33096, 32694, 33096, 33096, 32776, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=2 budget=40960 expected_recv=65406 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=40960 expected_recv=65597 expected_overflow=0 exact_recv_per_rank=[32799, 30771, 33096, 33095, 33095, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=40960 expected_recv=63868 expected_overflow=0 exact_recv_per_rank=[32799, 30771, 33096, 33095, 33095, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=2 budget=40960 expected_recv=65597 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=2 budget=40960 expected_recv=63868 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32799, 30771, 33096, 33095, 33095, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=40960 expected_recv=63866 expected_overflow=0 exact_recv_per_rank=[32799, 30771, 33096, 33095, 33095, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32799, 30771, 33096, 33095, 33095, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=2 budget=40960 expected_recv=63866 actual_valid_recv=64512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32799, 30771, 33096, 33095, 33095, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[33096, 31421, 32485, 33096, 33096, 33095, 33096, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=40960 expected_recv=66190 expected_overflow=0 exact_recv_per_rank=[32799, 30771, 33096, 33095, 33095, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=67200 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=64512 slack_slots=17408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=2 budget=40960 expected_recv=66190 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=40960 expected_recv=65953 expected_overflow=0 exact_recv_per_rank=[33096, 31421, 32485, 33096, 33096, 33095, 33096, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=40960 expected_recv=63112 expected_overflow=0 exact_recv_per_rank=[33096, 31421, 32485, 33096, 33096, 33095, 33096, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=67200 slack_slots=14720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=2 budget=40960 expected_recv=65953 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=2 budget=40960 expected_recv=63112 actual_valid_recv=63872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=40960 expected_recv=64945 expected_overflow=0 exact_recv_per_rank=[33096, 31421, 32485, 33096, 33096, 33095, 33096, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=63872 slack_slots=18048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=2 budget=40960 expected_recv=64945 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31421, 32485, 33096, 33096, 33095, 33096, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=40960 expected_recv=65511 expected_overflow=0 exact_recv_per_rank=[33096, 31421, 32485, 33096, 33096, 33095, 33096, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31421, 32485, 33096, 33096, 33095, 33096, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 31421, 32485, 33096, 33096, 33095, 33096, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65541 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 33094, 33096, 31796, 32770, 32779, 32812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=2 budget=40960 expected_recv=65511 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65541 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 33094, 33096, 31796, 32770, 32779, 32812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65309 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 33094, 33096, 31796, 32770, 32779, 32812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65309 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 33094, 33096, 31796, 32770, 32779, 32812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65468 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 33094, 33096, 31796, 32770, 32779, 32812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65436 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 33094, 33096, 31796, 32770, 32779, 32812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=40960 expected_recv=62992 expected_overflow=0 exact_recv_per_rank=[32784, 33096, 33096, 33096, 32759, 31481, 33096, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=40960 expected_recv=66191 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 33094, 33096, 31796, 32770, 32779, 32812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=40960 expected_recv=63960 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 33094, 33096, 31796, 32770, 32779, 32812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65468 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=2 budget=40960 expected_recv=65436 actual_valid_recv=66432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=12 calls=2 budget=40960 expected_recv=62992 actual_valid_recv=63744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=2 budget=40960 expected_recv=66191 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66432 slack_slots=15488 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=2 budget=40960 expected_recv=63960 actual_valid_recv=64896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32784, 33096, 33096, 33096, 32759, 31481, 33096, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65637 expected_overflow=0 exact_recv_per_rank=[32784, 33096, 33096, 33096, 32759, 31481, 33096, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=63744 slack_slots=18176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=2 received_num_tokens=81920 valid_num_tokens=64896 slack_slots=17024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32784, 33096, 33096, 33096, 32759, 31481, 33096, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65637 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32784, 33096, 33096, 33096, 32759, 31481, 33096, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65383 expected_overflow=0 exact_recv_per_rank=[32784, 33096, 33096, 33096, 32759, 31481, 33096, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65567 expected_overflow=0 exact_recv_per_rank=[33096, 32490, 32804, 32425, 33096, 32769, 32768, 32696] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32784, 33096, 33096, 33096, 32759, 31481, 33096, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65508 expected_overflow=0 exact_recv_per_rank=[32784, 33096, 33096, 33096, 32759, 31481, 33096, 32736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65383 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65567 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65000 expected_overflow=0 exact_recv_per_rank=[33096, 32490, 32804, 32425, 33096, 32769, 32768, 32696] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32490, 32804, 32425, 33096, 32769, 32768, 32696] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=12 calls=2 budget=40960 expected_recv=65508 actual_valid_recv=66176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65827 expected_overflow=0 exact_recv_per_rank=[33096, 32490, 32804, 32425, 33096, 32769, 32768, 32696] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65000 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=12 calls=2 received_num_tokens=81920 valid_num_tokens=66176 slack_slots=15744 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65827 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65529 expected_overflow=0 exact_recv_per_rank=[33096, 32490, 32804, 32425, 33096, 32769, 32768, 32696] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=40960 expected_recv=65550 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 31821, 32449, 33096, 32770, 33051, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65185 expected_overflow=0 exact_recv_per_rank=[33096, 32490, 32804, 32425, 33096, 32769, 32768, 32696] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=40960 expected_recv=64796 expected_overflow=0 exact_recv_per_rank=[33096, 32490, 32804, 32425, 33096, 32769, 32768, 32696] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65529 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=14 calls=2 budget=40960 expected_recv=65550 actual_valid_recv=66048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=13 calls=2 budget=40960 expected_recv=65185 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32490, 32804, 32425, 33096, 32769, 32768, 32696] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=2 budget=40960 expected_recv=64796 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 31821, 32449, 33096, 32770, 33051, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66048 slack_slots=15872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=40960 expected_recv=64441 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 31821, 32449, 33096, 32770, 33051, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=13 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=40960 expected_recv=64608 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 31821, 32449, 33096, 32770, 33051, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66944 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=2 budget=40960 expected_recv=64441 actual_valid_recv=65152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=13 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=2 budget=40960 expected_recv=64608 actual_valid_recv=65664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66944 slack_slots=14976 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65152 slack_slots=16768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65664 slack_slots=16256 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=40960 expected_recv=66147 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 31821, 32449, 33096, 32770, 33051, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=40960 expected_recv=64797 expected_overflow=0 exact_recv_per_rank=[33096, 32529, 32874, 32768, 32743, 32611, 32768, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 31821, 32449, 33096, 32770, 33051, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=2 budget=40960 expected_recv=66147 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=15 calls=2 budget=40960 expected_recv=64797 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=40960 expected_recv=64966 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 31821, 32449, 33096, 32770, 33051, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66816 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[32765, 33096, 31821, 32449, 33096, 32770, 33051, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=14 calls=2 budget=40960 expected_recv=64966 actual_valid_recv=65408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66816 slack_slots=15104 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65544 expected_overflow=0 exact_recv_per_rank=[33096, 32529, 32874, 32768, 32743, 32611, 32768, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=40960 expected_recv=66192 expected_overflow=0 exact_recv_per_rank=[33096, 32529, 32874, 32768, 32743, 32611, 32768, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=14 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=66688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=65408 slack_slots=16512 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65703 expected_overflow=0 exact_recv_per_rank=[33096, 32529, 32874, 32768, 32743, 32611, 32768, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65544 actual_valid_recv=66560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=2 budget=40960 expected_recv=66192 actual_valid_recv=67072 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=14 calls=2 received_num_tokens=81920 valid_num_tokens=66688 slack_slots=15232 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66560 slack_slots=15360 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65703 actual_valid_recv=66304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=67072 slack_slots=14848 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=66304 slack_slots=15616 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65535 expected_overflow=0 exact_recv_per_rank=[33096, 32529, 32874, 32768, 32743, 32611, 32768, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[38749, 34381, 31623, 34609, 3989, 37761, 33074, 47958] avg=32768.0000 max=47958 max_rank=7 min=3989 min_rank=4 imbalance_ratio=1.463562 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65513 expected_overflow=0 exact_recv_per_rank=[33096, 32529, 32874, 32768, 32743, 32611, 32768, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65535 actual_valid_recv=65792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65513 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65531 expected_overflow=0 exact_recv_per_rank=[33096, 32529, 32874, 32768, 32743, 32611, 32768, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65792 slack_slots=16128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65473 expected_overflow=0 exact_recv_per_rank=[33096, 32529, 32874, 32768, 32743, 32611, 32768, 32755] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65531 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=2 budget=40960 expected_recv=65473 actual_valid_recv=65920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[33095, 33096, 32908, 33096, 30684, 33096, 33074, 33095] avg=32768.0000 max=33096 max_rank=1 min=30684 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14862 max_load_reduction_pct=0.309896 moved_to_replicas=57329 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[38749, 34381, 31623, 34609, 3989, 37761, 33074, 47958] avg=32768.0000 max=47958 max_rank=7 min=3989 min_rank=4 imbalance_ratio=1.463562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[38749, 34381, 31623, 34609, 3989, 37761, 33074, 47958] avg=32768.0000 max=47958 max_rank=7 min=3989 min_rank=4 imbalance_ratio=1.463562 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=15 calls=2 received_num_tokens=81920 valid_num_tokens=65920 slack_slots=16000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[38749, 34381, 31623, 34609, 3989, 37761, 33074, 47958] avg=32768.0000 max=47958 max_rank=7 min=3989 min_rank=4 imbalance_ratio=1.463562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[38749, 34381, 31623, 34609, 3989, 37761, 33074, 47958] avg=32768.0000 max=47958 max_rank=7 min=3989 min_rank=4 imbalance_ratio=1.463562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[33095, 33096, 32908, 33096, 30684, 33096, 33074, 33095] avg=32768.0000 max=33096 max_rank=1 min=30684 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14862 max_load_reduction_pct=0.309896 moved_to_replicas=57329 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[33095, 33096, 32908, 33096, 30684, 33096, 33074, 33095] avg=32768.0000 max=33096 max_rank=1 min=30684 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14862 max_load_reduction_pct=0.309896 moved_to_replicas=57329 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[38749, 34381, 31623, 34609, 3989, 37761, 33074, 47958] avg=32768.0000 max=47958 max_rank=7 min=3989 min_rank=4 imbalance_ratio=1.463562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[41920, 6081, 130567, 49586, 32765, 5, 101, 1119] avg=32768.0000 max=130567 max_rank=2 min=5 min_rank=5 imbalance_ratio=3.984589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[33095, 33096, 32908, 33096, 30684, 33096, 33074, 33095] avg=32768.0000 max=33096 max_rank=1 min=30684 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14862 max_load_reduction_pct=0.309896 moved_to_replicas=57329 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[38749, 34381, 31623, 34609, 3989, 37761, 33074, 47958] avg=32768.0000 max=47958 max_rank=7 min=3989 min_rank=4 imbalance_ratio=1.463562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[38749, 34381, 31623, 34609, 3989, 37761, 33074, 47958] avg=32768.0000 max=47958 max_rank=7 min=3989 min_rank=4 imbalance_ratio=1.463562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[33095, 33096, 32908, 33096, 30684, 33096, 33074, 33095] avg=32768.0000 max=33096 max_rank=1 min=30684 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14862 max_load_reduction_pct=0.309896 moved_to_replicas=57329 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[33095, 33096, 32908, 33096, 30684, 33096, 33074, 33095] avg=32768.0000 max=33096 max_rank=1 min=30684 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14862 max_load_reduction_pct=0.309896 moved_to_replicas=57329 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[33095, 31396, 33096, 33096, 32765, 32760, 32853, 33083] avg=32768.0000 max=33096 max_rank=2 min=31396 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97471 max_load_reduction_pct=0.746521 moved_to_replicas=245320 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[41920, 6081, 130567, 49586, 32765, 5, 101, 1119] avg=32768.0000 max=130567 max_rank=2 min=5 min_rank=5 imbalance_ratio=3.984589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[41920, 6081, 130567, 49586, 32765, 5, 101, 1119] avg=32768.0000 max=130567 max_rank=2 min=5 min_rank=5 imbalance_ratio=3.984589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[33095, 33096, 32908, 33096, 30684, 33096, 33074, 33095] avg=32768.0000 max=33096 max_rank=1 min=30684 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14862 max_load_reduction_pct=0.309896 moved_to_replicas=57329 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[41920, 6081, 130567, 49586, 32765, 5, 101, 1119] avg=32768.0000 max=130567 max_rank=2 min=5 min_rank=5 imbalance_ratio=3.984589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[33095, 33096, 32908, 33096, 30684, 33096, 33074, 33095] avg=32768.0000 max=33096 max_rank=1 min=30684 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=14862 max_load_reduction_pct=0.309896 moved_to_replicas=57329 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[41920, 6081, 130567, 49586, 32765, 5, 101, 1119] avg=32768.0000 max=130567 max_rank=2 min=5 min_rank=5 imbalance_ratio=3.984589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[33095, 31396, 33096, 33096, 32765, 32760, 32853, 33083] avg=32768.0000 max=33096 max_rank=2 min=31396 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97471 max_load_reduction_pct=0.746521 moved_to_replicas=245320 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[33095, 31396, 33096, 33096, 32765, 32760, 32853, 33083] avg=32768.0000 max=33096 max_rank=2 min=31396 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97471 max_load_reduction_pct=0.746521 moved_to_replicas=245320 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[34026, 32782, 52239, 32762, 14, 1375, 43601, 65345] avg=32768.0000 max=65345 max_rank=7 min=14 min_rank=4 imbalance_ratio=1.994171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[41920, 6081, 130567, 49586, 32765, 5, 101, 1119] avg=32768.0000 max=130567 max_rank=2 min=5 min_rank=5 imbalance_ratio=3.984589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[33095, 31396, 33096, 33096, 32765, 32760, 32853, 33083] avg=32768.0000 max=33096 max_rank=2 min=31396 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97471 max_load_reduction_pct=0.746521 moved_to_replicas=245320 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[41920, 6081, 130567, 49586, 32765, 5, 101, 1119] avg=32768.0000 max=130567 max_rank=2 min=5 min_rank=5 imbalance_ratio=3.984589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[41920, 6081, 130567, 49586, 32765, 5, 101, 1119] avg=32768.0000 max=130567 max_rank=2 min=5 min_rank=5 imbalance_ratio=3.984589 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[33095, 31396, 33096, 33096, 32765, 32760, 32853, 33083] avg=32768.0000 max=33096 max_rank=2 min=31396 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97471 max_load_reduction_pct=0.746521 moved_to_replicas=245320 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[33096, 32782, 33096, 32762, 32263, 31953, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31953 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32249 max_load_reduction_pct=0.493519 moved_to_replicas=124619 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[34026, 32782, 52239, 32762, 14, 1375, 43601, 65345] avg=32768.0000 max=65345 max_rank=7 min=14 min_rank=4 imbalance_ratio=1.994171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[33095, 31396, 33096, 33096, 32765, 32760, 32853, 33083] avg=32768.0000 max=33096 max_rank=2 min=31396 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97471 max_load_reduction_pct=0.746521 moved_to_replicas=245320 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[33095, 31396, 33096, 33096, 32765, 32760, 32853, 33083] avg=32768.0000 max=33096 max_rank=2 min=31396 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97471 max_load_reduction_pct=0.746521 moved_to_replicas=245320 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[34026, 32782, 52239, 32762, 14, 1375, 43601, 65345] avg=32768.0000 max=65345 max_rank=7 min=14 min_rank=4 imbalance_ratio=1.994171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[33095, 31396, 33096, 33096, 32765, 32760, 32853, 33083] avg=32768.0000 max=33096 max_rank=2 min=31396 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97471 max_load_reduction_pct=0.746521 moved_to_replicas=245320 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[34026, 32782, 52239, 32762, 14, 1375, 43601, 65345] avg=32768.0000 max=65345 max_rank=7 min=14 min_rank=4 imbalance_ratio=1.994171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[33096, 32782, 33096, 32762, 32263, 31953, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31953 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32249 max_load_reduction_pct=0.493519 moved_to_replicas=124619 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[34026, 32782, 52239, 32762, 14, 1375, 43601, 65345] avg=32768.0000 max=65345 max_rank=7 min=14 min_rank=4 imbalance_ratio=1.994171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[33096, 32782, 33096, 32762, 32263, 31953, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31953 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32249 max_load_reduction_pct=0.493519 moved_to_replicas=124619 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[7, 32871, 23171, 6, 97115, 34963, 38737, 35274] avg=32768.0000 max=97115 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.963715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[34026, 32782, 52239, 32762, 14, 1375, 43601, 65345] avg=32768.0000 max=65345 max_rank=7 min=14 min_rank=4 imbalance_ratio=1.994171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[34026, 32782, 52239, 32762, 14, 1375, 43601, 65345] avg=32768.0000 max=65345 max_rank=7 min=14 min_rank=4 imbalance_ratio=1.994171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[33096, 32782, 33096, 32762, 32263, 31953, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31953 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32249 max_load_reduction_pct=0.493519 moved_to_replicas=124619 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[34026, 32782, 52239, 32762, 14, 1375, 43601, 65345] avg=32768.0000 max=65345 max_rank=7 min=14 min_rank=4 imbalance_ratio=1.994171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[33096, 32782, 33096, 32762, 32263, 31953, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31953 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32249 max_load_reduction_pct=0.493519 moved_to_replicas=124619 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[31294, 32871, 32856, 32738, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31294 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=64018 max_load_reduction_pct=0.659198 moved_to_replicas=149098 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[7, 32871, 23171, 6, 97115, 34963, 38737, 35274] avg=32768.0000 max=97115 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.963715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[33096, 32782, 33096, 32762, 32263, 31953, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31953 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32249 max_load_reduction_pct=0.493519 moved_to_replicas=124619 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[7, 32871, 23171, 6, 97115, 34963, 38737, 35274] avg=32768.0000 max=97115 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.963715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[33096, 32782, 33096, 32762, 32263, 31953, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31953 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32249 max_load_reduction_pct=0.493519 moved_to_replicas=124619 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[33096, 32782, 33096, 32762, 32263, 31953, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31953 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32249 max_load_reduction_pct=0.493519 moved_to_replicas=124619 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[7, 32871, 23171, 6, 97115, 34963, 38737, 35274] avg=32768.0000 max=97115 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.963715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[31294, 32871, 32856, 32738, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31294 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=64018 max_load_reduction_pct=0.659198 moved_to_replicas=149098 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[7, 32871, 23171, 6, 97115, 34963, 38737, 35274] avg=32768.0000 max=97115 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.963715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[68, 65391, 40155, 47625, 22242, 21462, 32762, 32439] avg=32768.0000 max=65391 max_rank=1 min=68 min_rank=0 imbalance_ratio=1.995575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[31294, 32871, 32856, 32738, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31294 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=64018 max_load_reduction_pct=0.659198 moved_to_replicas=149098 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[31294, 32871, 32856, 32738, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31294 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=64018 max_load_reduction_pct=0.659198 moved_to_replicas=149098 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[7, 32871, 23171, 6, 97115, 34963, 38737, 35274] avg=32768.0000 max=97115 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.963715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[7, 32871, 23171, 6, 97115, 34963, 38737, 35274] avg=32768.0000 max=97115 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.963715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[7, 32871, 23171, 6, 97115, 34963, 38737, 35274] avg=32768.0000 max=97115 max_rank=4 min=6 min_rank=3 imbalance_ratio=2.963715 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[31294, 32871, 32856, 32738, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31294 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=64018 max_load_reduction_pct=0.659198 moved_to_replicas=149098 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[32363, 33096, 33096, 33097, 32196, 33095, 32762, 32439] avg=32768.0000 max=33097 max_rank=3 min=32196 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32294 max_load_reduction_pct=0.493860 moved_to_replicas=109088 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[68, 65391, 40155, 47625, 22242, 21462, 32762, 32439] avg=32768.0000 max=65391 max_rank=1 min=68 min_rank=0 imbalance_ratio=1.995575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[68, 65391, 40155, 47625, 22242, 21462, 32762, 32439] avg=32768.0000 max=65391 max_rank=1 min=68 min_rank=0 imbalance_ratio=1.995575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[31294, 32871, 32856, 32738, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31294 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=64018 max_load_reduction_pct=0.659198 moved_to_replicas=149098 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[31294, 32871, 32856, 32738, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31294 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=64018 max_load_reduction_pct=0.659198 moved_to_replicas=149098 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[31294, 32871, 32856, 32738, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31294 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=64018 max_load_reduction_pct=0.659198 moved_to_replicas=149098 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[68, 65391, 40155, 47625, 22242, 21462, 32762, 32439] avg=32768.0000 max=65391 max_rank=1 min=68 min_rank=0 imbalance_ratio=1.995575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[32363, 33096, 33096, 33097, 32196, 33095, 32762, 32439] avg=32768.0000 max=33097 max_rank=3 min=32196 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32294 max_load_reduction_pct=0.493860 moved_to_replicas=109088 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[65536, 75070, 32212, 53140, 70, 32766, 3346, 4] avg=32768.0000 max=75070 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.290955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[68, 65391, 40155, 47625, 22242, 21462, 32762, 32439] avg=32768.0000 max=65391 max_rank=1 min=68 min_rank=0 imbalance_ratio=1.995575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[32363, 33096, 33096, 33097, 32196, 33095, 32762, 32439] avg=32768.0000 max=33097 max_rank=3 min=32196 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32294 max_load_reduction_pct=0.493860 moved_to_replicas=109088 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[32363, 33096, 33096, 33097, 32196, 33095, 32762, 32439] avg=32768.0000 max=33097 max_rank=3 min=32196 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32294 max_load_reduction_pct=0.493860 moved_to_replicas=109088 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[68, 65391, 40155, 47625, 22242, 21462, 32762, 32439] avg=32768.0000 max=65391 max_rank=1 min=68 min_rank=0 imbalance_ratio=1.995575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[68, 65391, 40155, 47625, 22242, 21462, 32762, 32439] avg=32768.0000 max=65391 max_rank=1 min=68 min_rank=0 imbalance_ratio=1.995575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[33096, 33096, 32212, 33097, 32019, 32766, 33096, 32762] avg=32768.0000 max=33097 max_rank=3 min=32019 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=41973 max_load_reduction_pct=0.559118 moved_to_replicas=187234 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[68, 65391, 40155, 47625, 22242, 21462, 32762, 32439] avg=32768.0000 max=65391 max_rank=1 min=68 min_rank=0 imbalance_ratio=1.995575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[32363, 33096, 33096, 33097, 32196, 33095, 32762, 32439] avg=32768.0000 max=33097 max_rank=3 min=32196 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32294 max_load_reduction_pct=0.493860 moved_to_replicas=109088 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[65536, 75070, 32212, 53140, 70, 32766, 3346, 4] avg=32768.0000 max=75070 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.290955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[65536, 75070, 32212, 53140, 70, 32766, 3346, 4] avg=32768.0000 max=75070 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.290955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[32363, 33096, 33096, 33097, 32196, 33095, 32762, 32439] avg=32768.0000 max=33097 max_rank=3 min=32196 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32294 max_load_reduction_pct=0.493860 moved_to_replicas=109088 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[32363, 33096, 33096, 33097, 32196, 33095, 32762, 32439] avg=32768.0000 max=33097 max_rank=3 min=32196 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32294 max_load_reduction_pct=0.493860 moved_to_replicas=109088 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[32363, 33096, 33096, 33097, 32196, 33095, 32762, 32439] avg=32768.0000 max=33097 max_rank=3 min=32196 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32294 max_load_reduction_pct=0.493860 moved_to_replicas=109088 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[65536, 75070, 32212, 53140, 70, 32766, 3346, 4] avg=32768.0000 max=75070 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.290955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[33096, 33096, 32212, 33097, 32019, 32766, 33096, 32762] avg=32768.0000 max=33097 max_rank=3 min=32019 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=41973 max_load_reduction_pct=0.559118 moved_to_replicas=187234 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[32763, 5704, 3851, 88542, 65517, 33001, 6, 32760] avg=32768.0000 max=88542 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.702087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[65536, 75070, 32212, 53140, 70, 32766, 3346, 4] avg=32768.0000 max=75070 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.290955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[33096, 33096, 32212, 33097, 32019, 32766, 33096, 32762] avg=32768.0000 max=33097 max_rank=3 min=32019 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=41973 max_load_reduction_pct=0.559118 moved_to_replicas=187234 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[33096, 33096, 32212, 33097, 32019, 32766, 33096, 32762] avg=32768.0000 max=33097 max_rank=3 min=32019 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=41973 max_load_reduction_pct=0.559118 moved_to_replicas=187234 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[65536, 75070, 32212, 53140, 70, 32766, 3346, 4] avg=32768.0000 max=75070 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.290955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[65536, 75070, 32212, 53140, 70, 32766, 3346, 4] avg=32768.0000 max=75070 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.290955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[32763, 33094, 31564, 33096, 33096, 33001, 32770, 32760] avg=32768.0000 max=33096 max_rank=3 min=31564 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55446 max_load_reduction_pct=0.626211 moved_to_replicas=174536 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[65536, 75070, 32212, 53140, 70, 32766, 3346, 4] avg=32768.0000 max=75070 max_rank=1 min=4 min_rank=7 imbalance_ratio=2.290955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[33096, 33096, 32212, 33097, 32019, 32766, 33096, 32762] avg=32768.0000 max=33097 max_rank=3 min=32019 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=41973 max_load_reduction_pct=0.559118 moved_to_replicas=187234 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[32763, 5704, 3851, 88542, 65517, 33001, 6, 32760] avg=32768.0000 max=88542 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.702087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[32763, 5704, 3851, 88542, 65517, 33001, 6, 32760] avg=32768.0000 max=88542 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.702087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[33096, 33096, 32212, 33097, 32019, 32766, 33096, 32762] avg=32768.0000 max=33097 max_rank=3 min=32019 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=41973 max_load_reduction_pct=0.559118 moved_to_replicas=187234 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[33096, 33096, 32212, 33097, 32019, 32766, 33096, 32762] avg=32768.0000 max=33097 max_rank=3 min=32019 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=41973 max_load_reduction_pct=0.559118 moved_to_replicas=187234 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[33096, 33096, 32212, 33097, 32019, 32766, 33096, 32762] avg=32768.0000 max=33097 max_rank=3 min=32019 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=41973 max_load_reduction_pct=0.559118 moved_to_replicas=187234 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[32763, 5704, 3851, 88542, 65517, 33001, 6, 32760] avg=32768.0000 max=88542 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.702087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[61291, 32114, 64960, 5714, 65532, 89, 32430, 14] avg=32768.0000 max=65532 max_rank=4 min=14 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[32763, 33094, 31564, 33096, 33096, 33001, 32770, 32760] avg=32768.0000 max=33096 max_rank=3 min=31564 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55446 max_load_reduction_pct=0.626211 moved_to_replicas=174536 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[32763, 5704, 3851, 88542, 65517, 33001, 6, 32760] avg=32768.0000 max=88542 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.702087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[32763, 33094, 31564, 33096, 33096, 33001, 32770, 32760] avg=32768.0000 max=33096 max_rank=3 min=31564 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55446 max_load_reduction_pct=0.626211 moved_to_replicas=174536 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[32763, 33094, 31564, 33096, 33096, 33001, 32770, 32760] avg=32768.0000 max=33096 max_rank=3 min=31564 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55446 max_load_reduction_pct=0.626211 moved_to_replicas=174536 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[32763, 5704, 3851, 88542, 65517, 33001, 6, 32760] avg=32768.0000 max=88542 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.702087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[33098, 32114, 33096, 33094, 33096, 32766, 32430, 32450] avg=32768.0000 max=33098 max_rank=0 min=32114 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=185194 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[32763, 5704, 3851, 88542, 65517, 33001, 6, 32760] avg=32768.0000 max=88542 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.702087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[32763, 5704, 3851, 88542, 65517, 33001, 6, 32760] avg=32768.0000 max=88542 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.702087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[32763, 33094, 31564, 33096, 33096, 33001, 32770, 32760] avg=32768.0000 max=33096 max_rank=3 min=31564 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55446 max_load_reduction_pct=0.626211 moved_to_replicas=174536 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[61291, 32114, 64960, 5714, 65532, 89, 32430, 14] avg=32768.0000 max=65532 max_rank=4 min=14 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[61291, 32114, 64960, 5714, 65532, 89, 32430, 14] avg=32768.0000 max=65532 max_rank=4 min=14 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[32763, 33094, 31564, 33096, 33096, 33001, 32770, 32760] avg=32768.0000 max=33096 max_rank=3 min=31564 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55446 max_load_reduction_pct=0.626211 moved_to_replicas=174536 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[32763, 33094, 31564, 33096, 33096, 33001, 32770, 32760] avg=32768.0000 max=33096 max_rank=3 min=31564 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55446 max_load_reduction_pct=0.626211 moved_to_replicas=174536 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[32763, 33094, 31564, 33096, 33096, 33001, 32770, 32760] avg=32768.0000 max=33096 max_rank=3 min=31564 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55446 max_load_reduction_pct=0.626211 moved_to_replicas=174536 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[61291, 32114, 64960, 5714, 65532, 89, 32430, 14] avg=32768.0000 max=65532 max_rank=4 min=14 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[25453, 300, 98292, 32694, 43447, 24988, 12, 36958] avg=32768.0000 max=98292 max_rank=2 min=12 min_rank=6 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[33098, 32114, 33096, 33094, 33096, 32766, 32430, 32450] avg=32768.0000 max=33098 max_rank=0 min=32114 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=185194 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[33098, 32114, 33096, 33094, 33096, 32766, 32430, 32450] avg=32768.0000 max=33098 max_rank=0 min=32114 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=185194 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[61291, 32114, 64960, 5714, 65532, 89, 32430, 14] avg=32768.0000 max=65532 max_rank=4 min=14 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[61291, 32114, 64960, 5714, 65532, 89, 32430, 14] avg=32768.0000 max=65532 max_rank=4 min=14 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[33098, 32114, 33096, 33094, 33096, 32766, 32430, 32450] avg=32768.0000 max=33098 max_rank=0 min=32114 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=185194 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[31558, 32732, 33096, 32694, 33096, 33096, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=31558 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65196 max_load_reduction_pct=0.663289 moved_to_replicas=155561 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[61291, 32114, 64960, 5714, 65532, 89, 32430, 14] avg=32768.0000 max=65532 max_rank=4 min=14 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[61291, 32114, 64960, 5714, 65532, 89, 32430, 14] avg=32768.0000 max=65532 max_rank=4 min=14 min_rank=7 imbalance_ratio=1.999878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[33098, 32114, 33096, 33094, 33096, 32766, 32430, 32450] avg=32768.0000 max=33098 max_rank=0 min=32114 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=185194 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[25453, 300, 98292, 32694, 43447, 24988, 12, 36958] avg=32768.0000 max=98292 max_rank=2 min=12 min_rank=6 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[25453, 300, 98292, 32694, 43447, 24988, 12, 36958] avg=32768.0000 max=98292 max_rank=2 min=12 min_rank=6 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[33098, 32114, 33096, 33094, 33096, 32766, 32430, 32450] avg=32768.0000 max=33098 max_rank=0 min=32114 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=185194 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[33098, 32114, 33096, 33094, 33096, 32766, 32430, 32450] avg=32768.0000 max=33098 max_rank=0 min=32114 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=185194 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[33098, 32114, 33096, 33094, 33096, 32766, 32430, 32450] avg=32768.0000 max=33098 max_rank=0 min=32114 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=32434 max_load_reduction_pct=0.494934 moved_to_replicas=185194 active_clone_links=8 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[25453, 300, 98292, 32694, 43447, 24988, 12, 36958] avg=32768.0000 max=98292 max_rank=2 min=12 min_rank=6 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[42, 1162, 64957, 9129, 54831, 34662, 18610, 78751] avg=32768.0000 max=78751 max_rank=7 min=42 min_rank=0 imbalance_ratio=2.403290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[31558, 32732, 33096, 32694, 33096, 33096, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=31558 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65196 max_load_reduction_pct=0.663289 moved_to_replicas=155561 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[31558, 32732, 33096, 32694, 33096, 33096, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=31558 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65196 max_load_reduction_pct=0.663289 moved_to_replicas=155561 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[25453, 300, 98292, 32694, 43447, 24988, 12, 36958] avg=32768.0000 max=98292 max_rank=2 min=12 min_rank=6 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[25453, 300, 98292, 32694, 43447, 24988, 12, 36958] avg=32768.0000 max=98292 max_rank=2 min=12 min_rank=6 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[31558, 32732, 33096, 32694, 33096, 33096, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=31558 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65196 max_load_reduction_pct=0.663289 moved_to_replicas=155561 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[32799, 30771, 33096, 33095, 33095, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=30771 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=45655 max_load_reduction_pct=0.579739 moved_to_replicas=200943 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[25453, 300, 98292, 32694, 43447, 24988, 12, 36958] avg=32768.0000 max=98292 max_rank=2 min=12 min_rank=6 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[25453, 300, 98292, 32694, 43447, 24988, 12, 36958] avg=32768.0000 max=98292 max_rank=2 min=12 min_rank=6 imbalance_ratio=2.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[31558, 32732, 33096, 32694, 33096, 33096, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=31558 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65196 max_load_reduction_pct=0.663289 moved_to_replicas=155561 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[42, 1162, 64957, 9129, 54831, 34662, 18610, 78751] avg=32768.0000 max=78751 max_rank=7 min=42 min_rank=0 imbalance_ratio=2.403290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[31558, 32732, 33096, 32694, 33096, 33096, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=31558 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65196 max_load_reduction_pct=0.663289 moved_to_replicas=155561 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[42, 1162, 64957, 9129, 54831, 34662, 18610, 78751] avg=32768.0000 max=78751 max_rank=7 min=42 min_rank=0 imbalance_ratio=2.403290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[31558, 32732, 33096, 32694, 33096, 33096, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=31558 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65196 max_load_reduction_pct=0.663289 moved_to_replicas=155561 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[31558, 32732, 33096, 32694, 33096, 33096, 32776, 33096] avg=32768.0000 max=33096 max_rank=2 min=31558 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=65196 max_load_reduction_pct=0.663289 moved_to_replicas=155561 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[42, 1162, 64957, 9129, 54831, 34662, 18610, 78751] avg=32768.0000 max=78751 max_rank=7 min=42 min_rank=0 imbalance_ratio=2.403290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[32799, 30771, 33096, 33095, 33095, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=30771 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=45655 max_load_reduction_pct=0.579739 moved_to_replicas=200943 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[33453, 18468, 48, 65222, 65533, 13353, 33308, 32759] avg=32768.0000 max=65533 max_rank=4 min=48 min_rank=2 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[32799, 30771, 33096, 33095, 33095, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=30771 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=45655 max_load_reduction_pct=0.579739 moved_to_replicas=200943 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[42, 1162, 64957, 9129, 54831, 34662, 18610, 78751] avg=32768.0000 max=78751 max_rank=7 min=42 min_rank=0 imbalance_ratio=2.403290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[42, 1162, 64957, 9129, 54831, 34662, 18610, 78751] avg=32768.0000 max=78751 max_rank=7 min=42 min_rank=0 imbalance_ratio=2.403290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[32799, 30771, 33096, 33095, 33095, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=30771 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=45655 max_load_reduction_pct=0.579739 moved_to_replicas=200943 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[33096, 31421, 32485, 33096, 33096, 33095, 33096, 32759] avg=32768.0000 max=33096 max_rank=0 min=31421 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=129806 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[42, 1162, 64957, 9129, 54831, 34662, 18610, 78751] avg=32768.0000 max=78751 max_rank=7 min=42 min_rank=0 imbalance_ratio=2.403290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[42, 1162, 64957, 9129, 54831, 34662, 18610, 78751] avg=32768.0000 max=78751 max_rank=7 min=42 min_rank=0 imbalance_ratio=2.403290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[32799, 30771, 33096, 33095, 33095, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=30771 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=45655 max_load_reduction_pct=0.579739 moved_to_replicas=200943 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[33453, 18468, 48, 65222, 65533, 13353, 33308, 32759] avg=32768.0000 max=65533 max_rank=4 min=48 min_rank=2 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[32799, 30771, 33096, 33095, 33095, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=30771 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=45655 max_load_reduction_pct=0.579739 moved_to_replicas=200943 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[33453, 18468, 48, 65222, 65533, 13353, 33308, 32759] avg=32768.0000 max=65533 max_rank=4 min=48 min_rank=2 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[32799, 30771, 33096, 33095, 33095, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=30771 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=45655 max_load_reduction_pct=0.579739 moved_to_replicas=200943 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[32799, 30771, 33096, 33095, 33095, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=2 min=30771 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=45655 max_load_reduction_pct=0.579739 moved_to_replicas=200943 active_clone_links=14 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[32701, 66012, 57154, 74390, 31796, 6, 36, 49] avg=32768.0000 max=74390 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.270203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[33453, 18468, 48, 65222, 65533, 13353, 33308, 32759] avg=32768.0000 max=65533 max_rank=4 min=48 min_rank=2 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[33096, 31421, 32485, 33096, 33096, 33095, 33096, 32759] avg=32768.0000 max=33096 max_rank=0 min=31421 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=129806 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[33453, 18468, 48, 65222, 65533, 13353, 33308, 32759] avg=32768.0000 max=65533 max_rank=4 min=48 min_rank=2 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[33096, 31421, 32485, 33096, 33096, 33095, 33096, 32759] avg=32768.0000 max=33096 max_rank=0 min=31421 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=129806 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[32701, 33096, 33094, 33096, 31796, 32770, 32779, 32812] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=41294 max_load_reduction_pct=0.555101 moved_to_replicas=196109 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[33453, 18468, 48, 65222, 65533, 13353, 33308, 32759] avg=32768.0000 max=65533 max_rank=4 min=48 min_rank=2 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[33453, 18468, 48, 65222, 65533, 13353, 33308, 32759] avg=32768.0000 max=65533 max_rank=4 min=48 min_rank=2 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[33096, 31421, 32485, 33096, 33096, 33095, 33096, 32759] avg=32768.0000 max=33096 max_rank=0 min=31421 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=129806 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[33453, 18468, 48, 65222, 65533, 13353, 33308, 32759] avg=32768.0000 max=65533 max_rank=4 min=48 min_rank=2 imbalance_ratio=1.999908 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[33096, 31421, 32485, 33096, 33096, 33095, 33096, 32759] avg=32768.0000 max=33096 max_rank=0 min=31421 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=129806 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[32701, 66012, 57154, 74390, 31796, 6, 36, 49] avg=32768.0000 max=74390 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.270203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[33096, 31421, 32485, 33096, 33096, 33095, 33096, 32759] avg=32768.0000 max=33096 max_rank=0 min=31421 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=129806 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[33096, 31421, 32485, 33096, 33096, 33095, 33096, 32759] avg=32768.0000 max=33096 max_rank=0 min=31421 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=129806 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[32701, 66012, 57154, 74390, 31796, 6, 36, 49] avg=32768.0000 max=74390 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.270203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[20, 363, 66790, 35207, 32759, 28912, 65357, 32736] avg=32768.0000 max=66790 max_rank=2 min=20 min_rank=0 imbalance_ratio=2.038269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[33096, 31421, 32485, 33096, 33096, 33095, 33096, 32759] avg=32768.0000 max=33096 max_rank=0 min=31421 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32437 max_load_reduction_pct=0.494972 moved_to_replicas=129806 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[32701, 33096, 33094, 33096, 31796, 32770, 32779, 32812] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=41294 max_load_reduction_pct=0.555101 moved_to_replicas=196109 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[32701, 66012, 57154, 74390, 31796, 6, 36, 49] avg=32768.0000 max=74390 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.270203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[32701, 66012, 57154, 74390, 31796, 6, 36, 49] avg=32768.0000 max=74390 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.270203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[32701, 33096, 33094, 33096, 31796, 32770, 32779, 32812] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=41294 max_load_reduction_pct=0.555101 moved_to_replicas=196109 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[32701, 66012, 57154, 74390, 31796, 6, 36, 49] avg=32768.0000 max=74390 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.270203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[32784, 33096, 33096, 33096, 32759, 31481, 33096, 32736] avg=32768.0000 max=33096 max_rank=1 min=31481 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33694 max_load_reduction_pct=0.504477 moved_to_replicas=135563 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[32701, 66012, 57154, 74390, 31796, 6, 36, 49] avg=32768.0000 max=74390 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.270203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[32701, 33096, 33094, 33096, 31796, 32770, 32779, 32812] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=41294 max_load_reduction_pct=0.555101 moved_to_replicas=196109 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[32701, 66012, 57154, 74390, 31796, 6, 36, 49] avg=32768.0000 max=74390 max_rank=3 min=6 min_rank=5 imbalance_ratio=2.270203 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[32701, 33096, 33094, 33096, 31796, 32770, 32779, 32812] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=41294 max_load_reduction_pct=0.555101 moved_to_replicas=196109 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[20, 363, 66790, 35207, 32759, 28912, 65357, 32736] avg=32768.0000 max=66790 max_rank=2 min=20 min_rank=0 imbalance_ratio=2.038269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[32701, 33096, 33094, 33096, 31796, 32770, 32779, 32812] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=41294 max_load_reduction_pct=0.555101 moved_to_replicas=196109 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[32701, 33096, 33094, 33096, 31796, 32770, 32779, 32812] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=41294 max_load_reduction_pct=0.555101 moved_to_replicas=196109 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[32701, 33096, 33094, 33096, 31796, 32770, 32779, 32812] avg=32768.0000 max=33096 max_rank=1 min=31796 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=41294 max_load_reduction_pct=0.555101 moved_to_replicas=196109 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[20, 363, 66790, 35207, 32759, 28912, 65357, 32736] avg=32768.0000 max=66790 max_rank=2 min=20 min_rank=0 imbalance_ratio=2.038269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[64798, 788, 32804, 6, 65515, 32769, 32768, 32696] avg=32768.0000 max=65515 max_rank=4 min=6 min_rank=3 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[32784, 33096, 33096, 33096, 32759, 31481, 33096, 32736] avg=32768.0000 max=33096 max_rank=1 min=31481 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33694 max_load_reduction_pct=0.504477 moved_to_replicas=135563 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[20, 363, 66790, 35207, 32759, 28912, 65357, 32736] avg=32768.0000 max=66790 max_rank=2 min=20 min_rank=0 imbalance_ratio=2.038269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[20, 363, 66790, 35207, 32759, 28912, 65357, 32736] avg=32768.0000 max=66790 max_rank=2 min=20 min_rank=0 imbalance_ratio=2.038269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[20, 363, 66790, 35207, 32759, 28912, 65357, 32736] avg=32768.0000 max=66790 max_rank=2 min=20 min_rank=0 imbalance_ratio=2.038269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[32784, 33096, 33096, 33096, 32759, 31481, 33096, 32736] avg=32768.0000 max=33096 max_rank=1 min=31481 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33694 max_load_reduction_pct=0.504477 moved_to_replicas=135563 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32490, 32804, 32425, 33096, 32769, 32768, 32696] avg=32768.0000 max=33096 max_rank=0 min=32425 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=127965 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[20, 363, 66790, 35207, 32759, 28912, 65357, 32736] avg=32768.0000 max=66790 max_rank=2 min=20 min_rank=0 imbalance_ratio=2.038269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[32784, 33096, 33096, 33096, 32759, 31481, 33096, 32736] avg=32768.0000 max=33096 max_rank=1 min=31481 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33694 max_load_reduction_pct=0.504477 moved_to_replicas=135563 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[20, 363, 66790, 35207, 32759, 28912, 65357, 32736] avg=32768.0000 max=66790 max_rank=2 min=20 min_rank=0 imbalance_ratio=2.038269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[32784, 33096, 33096, 33096, 32759, 31481, 33096, 32736] avg=32768.0000 max=33096 max_rank=1 min=31481 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33694 max_load_reduction_pct=0.504477 moved_to_replicas=135563 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[64798, 788, 32804, 6, 65515, 32769, 32768, 32696] avg=32768.0000 max=65515 max_rank=4 min=6 min_rank=3 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[32784, 33096, 33096, 33096, 32759, 31481, 33096, 32736] avg=32768.0000 max=33096 max_rank=1 min=31481 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33694 max_load_reduction_pct=0.504477 moved_to_replicas=135563 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[32784, 33096, 33096, 33096, 32759, 31481, 33096, 32736] avg=32768.0000 max=33096 max_rank=1 min=31481 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33694 max_load_reduction_pct=0.504477 moved_to_replicas=135563 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[64798, 788, 32804, 6, 65515, 32769, 32768, 32696] avg=32768.0000 max=65515 max_rank=4 min=6 min_rank=3 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[4, 62629, 9, 352, 97669, 32770, 3518, 65193] avg=32768.0000 max=97669 max_rank=4 min=4 min_rank=0 imbalance_ratio=2.980621 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[32784, 33096, 33096, 33096, 32759, 31481, 33096, 32736] avg=32768.0000 max=33096 max_rank=1 min=31481 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=33694 max_load_reduction_pct=0.504477 moved_to_replicas=135563 active_clone_links=10 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32490, 32804, 32425, 33096, 32769, 32768, 32696] avg=32768.0000 max=33096 max_rank=0 min=32425 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=127965 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[64798, 788, 32804, 6, 65515, 32769, 32768, 32696] avg=32768.0000 max=65515 max_rank=4 min=6 min_rank=3 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[64798, 788, 32804, 6, 65515, 32769, 32768, 32696] avg=32768.0000 max=65515 max_rank=4 min=6 min_rank=3 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[64798, 788, 32804, 6, 65515, 32769, 32768, 32696] avg=32768.0000 max=65515 max_rank=4 min=6 min_rank=3 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32490, 32804, 32425, 33096, 32769, 32768, 32696] avg=32768.0000 max=33096 max_rank=0 min=32425 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=127965 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[32765, 33096, 31821, 32449, 33096, 32770, 33051, 33096] avg=32768.0000 max=33096 max_rank=1 min=31821 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64573 max_load_reduction_pct=0.661141 moved_to_replicas=251089 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[64798, 788, 32804, 6, 65515, 32769, 32768, 32696] avg=32768.0000 max=65515 max_rank=4 min=6 min_rank=3 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[64798, 788, 32804, 6, 65515, 32769, 32768, 32696] avg=32768.0000 max=65515 max_rank=4 min=6 min_rank=3 imbalance_ratio=1.999359 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32490, 32804, 32425, 33096, 32769, 32768, 32696] avg=32768.0000 max=33096 max_rank=0 min=32425 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=127965 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32490, 32804, 32425, 33096, 32769, 32768, 32696] avg=32768.0000 max=33096 max_rank=0 min=32425 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=127965 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32490, 32804, 32425, 33096, 32769, 32768, 32696] avg=32768.0000 max=33096 max_rank=0 min=32425 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=127965 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[4, 62629, 9, 352, 97669, 32770, 3518, 65193] avg=32768.0000 max=97669 max_rank=4 min=4 min_rank=0 imbalance_ratio=2.980621 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32490, 32804, 32425, 33096, 32769, 32768, 32696] avg=32768.0000 max=33096 max_rank=0 min=32425 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=127965 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[4, 62629, 9, 352, 97669, 32770, 3518, 65193] avg=32768.0000 max=97669 max_rank=4 min=4 min_rank=0 imbalance_ratio=2.980621 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[130323, 795, 32874, 32768, 3, 32611, 32768, 2] avg=32768.0000 max=130323 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.977142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[33096, 32490, 32804, 32425, 33096, 32769, 32768, 32696] avg=32768.0000 max=33096 max_rank=0 min=32425 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=32419 max_load_reduction_pct=0.494833 moved_to_replicas=127965 active_clone_links=4 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[32765, 33096, 31821, 32449, 33096, 32770, 33051, 33096] avg=32768.0000 max=33096 max_rank=1 min=31821 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64573 max_load_reduction_pct=0.661141 moved_to_replicas=251089 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[4, 62629, 9, 352, 97669, 32770, 3518, 65193] avg=32768.0000 max=97669 max_rank=4 min=4 min_rank=0 imbalance_ratio=2.980621 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[32765, 33096, 31821, 32449, 33096, 32770, 33051, 33096] avg=32768.0000 max=33096 max_rank=1 min=31821 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64573 max_load_reduction_pct=0.661141 moved_to_replicas=251089 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[4, 62629, 9, 352, 97669, 32770, 3518, 65193] avg=32768.0000 max=97669 max_rank=4 min=4 min_rank=0 imbalance_ratio=2.980621 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[4, 62629, 9, 352, 97669, 32770, 3518, 65193] avg=32768.0000 max=97669 max_rank=4 min=4 min_rank=0 imbalance_ratio=2.980621 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[33096, 32529, 32874, 32768, 32743, 32611, 32768, 32755] avg=32768.0000 max=33096 max_rank=0 min=32529 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97227 max_load_reduction_pct=0.746046 moved_to_replicas=194218 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[4, 62629, 9, 352, 97669, 32770, 3518, 65193] avg=32768.0000 max=97669 max_rank=4 min=4 min_rank=0 imbalance_ratio=2.980621 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[4, 62629, 9, 352, 97669, 32770, 3518, 65193] avg=32768.0000 max=97669 max_rank=4 min=4 min_rank=0 imbalance_ratio=2.980621 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[32765, 33096, 31821, 32449, 33096, 32770, 33051, 33096] avg=32768.0000 max=33096 max_rank=1 min=31821 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64573 max_load_reduction_pct=0.661141 moved_to_replicas=251089 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[32765, 33096, 31821, 32449, 33096, 32770, 33051, 33096] avg=32768.0000 max=33096 max_rank=1 min=31821 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64573 max_load_reduction_pct=0.661141 moved_to_replicas=251089 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[32765, 33096, 31821, 32449, 33096, 32770, 33051, 33096] avg=32768.0000 max=33096 max_rank=1 min=31821 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64573 max_load_reduction_pct=0.661141 moved_to_replicas=251089 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[130323, 795, 32874, 32768, 3, 32611, 32768, 2] avg=32768.0000 max=130323 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.977142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[130323, 795, 32874, 32768, 3, 32611, 32768, 2] avg=32768.0000 max=130323 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.977142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[32765, 33096, 31821, 32449, 33096, 32770, 33051, 33096] avg=32768.0000 max=33096 max_rank=1 min=31821 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64573 max_load_reduction_pct=0.661141 moved_to_replicas=251089 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[32765, 33096, 31821, 32449, 33096, 32770, 33051, 33096] avg=32768.0000 max=33096 max_rank=1 min=31821 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=64573 max_load_reduction_pct=0.661141 moved_to_replicas=251089 active_clone_links=9 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[33096, 32529, 32874, 32768, 32743, 32611, 32768, 32755] avg=32768.0000 max=33096 max_rank=0 min=32529 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97227 max_load_reduction_pct=0.746046 moved_to_replicas=194218 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[130323, 795, 32874, 32768, 3, 32611, 32768, 2] avg=32768.0000 max=130323 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.977142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[33096, 32529, 32874, 32768, 32743, 32611, 32768, 32755] avg=32768.0000 max=33096 max_rank=0 min=32529 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97227 max_load_reduction_pct=0.746046 moved_to_replicas=194218 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[130323, 795, 32874, 32768, 3, 32611, 32768, 2] avg=32768.0000 max=130323 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.977142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[130323, 795, 32874, 32768, 3, 32611, 32768, 2] avg=32768.0000 max=130323 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.977142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[130323, 795, 32874, 32768, 3, 32611, 32768, 2] avg=32768.0000 max=130323 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.977142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[130323, 795, 32874, 32768, 3, 32611, 32768, 2] avg=32768.0000 max=130323 max_rank=0 min=2 min_rank=7 imbalance_ratio=3.977142 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[33096, 32529, 32874, 32768, 32743, 32611, 32768, 32755] avg=32768.0000 max=33096 max_rank=0 min=32529 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97227 max_load_reduction_pct=0.746046 moved_to_replicas=194218 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[33096, 32529, 32874, 32768, 32743, 32611, 32768, 32755] avg=32768.0000 max=33096 max_rank=0 min=32529 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97227 max_load_reduction_pct=0.746046 moved_to_replicas=194218 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[33096, 32529, 32874, 32768, 32743, 32611, 32768, 32755] avg=32768.0000 max=33096 max_rank=0 min=32529 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97227 max_load_reduction_pct=0.746046 moved_to_replicas=194218 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[33096, 32529, 32874, 32768, 32743, 32611, 32768, 32755] avg=32768.0000 max=33096 max_rank=0 min=32529 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97227 max_load_reduction_pct=0.746046 moved_to_replicas=194218 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[33096, 32529, 32874, 32768, 32743, 32611, 32768, 32755] avg=32768.0000 max=33096 max_rank=0 min=32529 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=97227 max_load_reduction_pct=0.746046 moved_to_replicas=194218 active_clone_links=6 calls=2 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=32 total_expected_recv=1046159 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254336 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=32 total_expected_recv=1045619 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=252288 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=32 total_expected_recv=1047992 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=251520 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=32 total_expected_recv=1048853 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=249728 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=32 total_expected_recv=1050755 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=250624 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=32 total_expected_recv=1053650 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=247168 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=32 total_expected_recv=1050495 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=250496 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=32 total_expected_recv=1045085 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=254464 peak_demand=33096 required_capacity=1.010010 + [2026-08-30 05:10:32] iteration 12/ 12 | consumed samples: 192 | elapsed time per iteration (ms): 577.0 | throughput per GPU (TFLOP/s/GPU): 577.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.842852E+00 | loss scale: 1.0 | grad norm: 5.015 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +[after training is done] datetime: 2026-08-30 05:10:32 +[rank0]:[W830 05:10:33.881840347 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W830 05:10:35.483105168 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W830 05:10:35.489193106 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W830 05:10:35.490181218 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W830 05:10:35.492919433 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W830 05:10:35.496392915 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W830 05:10:35.497087968 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W830 05:10:35.515415159 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 05:10:38.494000 485280 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 05:10:38.494000 485280 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 485372 closing signal SIGTERM +W0830 05:10:38.495000 485280 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 485373 closing signal SIGTERM +W0830 05:10:38.495000 485280 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 485374 closing signal SIGTERM +W0830 05:10:38.496000 485280 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 485375 closing signal SIGTERM +W0830 05:10:38.496000 485280 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 485376 closing signal SIGTERM +W0830 05:10:38.497000 485280 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 485377 closing signal SIGTERM +W0830 05:10:38.497000 485280 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 485378 closing signal SIGTERM +W0830 05:10:38.498000 485280 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 485379 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 485280 got signal: 15 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/mem_samples.csv b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..50ccc31bba986f1669c414570af5bc44c2f01e8f --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/mem_samples.csv @@ -0,0 +1,152 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 629 +2, 629 +3, 629 +4, 629 +5, 629 +6, 629 +7, 629 +0, 913 +1, 903 +2, 917 +3, 933 +4, 929 +5, 933 +6, 941 +7, 925 +0, 1895 +1, 1895 +2, 1895 +3, 1895 +4, 1895 +5, 1895 +6, 1895 +7, 1895 +0, 2349 +1, 2351 +2, 2343 +3, 2329 +4, 2339 +5, 2353 +6, 2359 +7, 2359 +0, 42337 +1, 42333 +2, 42333 +3, 42333 +4, 42333 +5, 42333 +6, 42333 +7, 42333 +0, 42743 +1, 42733 +2, 42733 +3, 42733 +4, 42733 +5, 42733 +6, 42733 +7, 42733 +0, 44049 +1, 44041 +2, 44041 +3, 44041 +4, 44041 +5, 44041 +6, 44041 +7, 44041 +0, 44369 +1, 44361 +2, 44361 +3, 44361 +4, 44361 +5, 44361 +6, 44361 +7, 44361 +0, 44369 +1, 44361 +2, 44361 +3, 44361 +4, 44361 +5, 44361 +6, 44361 +7, 44361 +0, 46011 +1, 46003 +2, 46003 +3, 46003 +4, 46003 +5, 46003 +6, 46003 +7, 46003 +0, 72849 +1, 72841 +2, 72841 +3, 72841 +4, 72841 +5, 72841 +6, 72841 +7, 72841 +0, 72849 +1, 72841 +2, 72841 +3, 72841 +4, 72841 +5, 72841 +6, 72841 +7, 72841 +0, 73409 +1, 73401 +2, 73401 +3, 73401 +4, 73401 +5, 73401 +6, 73401 +7, 73401 +0, 84977 +1, 85037 +2, 85031 +3, 85039 +4, 84971 +5, 85043 +6, 85053 +7, 85041 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/run.env b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..a25beb3aeaba8028b001d4e30035ba5a15394160 --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/run.env @@ -0,0 +1,70 @@ +backend=ultraep +config=G2 +family=tierA +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=3584 +num_experts=32 +topk=8 +experts_per_rank=4 +expert_params=44040192 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=32768 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=4 +capacity_mode=static +capacity=1.25 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=1 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=1024 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=0 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/train.log b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..b32dfa1cfe3409fec77a80413ef4b2320e067a5e --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/train.log @@ -0,0 +1,1116 @@ +[05:11:57] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + G2 / ultraep (slots/rank=4 policy=matched) + E=32 K=8 H'=3584 m=4 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=static cap=1.25 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25__forcebal/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0830 05:11:58.814000 492296 torch/distributed/run.py:766] +W0830 05:11:58.814000 492296 torch/distributed/run.py:766] ***************************************** +W0830 05:11:58.814000 492296 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 05:11:58.814000 492296 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 3584 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 4 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.25 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. True + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 32 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank [Gloo] Rank 0 is connected to 77 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : [Gloo] Rank +72 + is connected to [Gloo] Rank 7 peer ranks. Expected number of connected peer ranks is : 17 is connected to +7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 75 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7[Gloo] Rank peer ranks. 6[Gloo] Rank Expected number of connected peer ranks is : is connected to 577 is connected to peer ranks. +7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank7]:[W830 05:12:17.285927019 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 05:12:17.286012001 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 05:12:17.286134516 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 05:12:17.286198702 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 05:12:17.286272222 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 05:12:17.286962301 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 05:12:17.287092983 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 05:12:17.455273505 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.741 seconds +time to initialize megatron (seconds): 7.181 +[after megatron is initialized] datetime: 2026-08-30 05:12:22 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 6722555904 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (42086400 elements, 42086400 padded size): + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.final_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight +Params for bucket 3 (42082304 elements, 42082304 padded size): + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_qkv.weight +Params for bucket 4 (42082304 elements, 42082304 padded size): + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_qkv.weight +Params for bucket 5 (42082304 elements, 42082304 padded size): + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.linear_qkv.weight +Params for bucket 6 (42082304 elements, 42082304 padded size): + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.pre_mlp_layernorm.weight +Params for bucket 7 (42082304 elements, 42082304 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.pre_mlp_layernorm.weight +Params for bucket 8 (42082304 elements, 42082304 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight +Params for bucket 9 (42082304 elements, 42082304 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.pre_mlp_layernorm.weight +Params for bucket 10 (42082304 elements, 42082304 padded size): + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_proj.weight +Params for bucket 11 (42082304 elements, 42082304 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.linear_qkv.weight +Params for bucket 12 (42082304 elements, 42082304 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight +Params for bucket 13 (42082304 elements, 42082304 padded size): + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight +Params for bucket 14 (42082304 elements, 42082304 padded size): + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight +Params for bucket 15 (42082304 elements, 42082304 padded size): + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight +Params for bucket 16 (42082304 elements, 42082304 padded size): + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight +Params for bucket 17 (42082304 elements, 42082304 padded size): + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 56 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 +Params for bucket 3 (58720256 elements, 58720256 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 +Params for bucket 5 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 +Params for bucket 6 (58720256 elements, 58720256 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 +Params for bucket 7 (58720256 elements, 58720256 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 +Params for bucket 9 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 +Params for bucket 10 (58720256 elements, 58720256 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 +Params for bucket 13 (58720256 elements, 58720256 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 +Params for bucket 14 (58720256 elements, 58720256 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 17 (58720256 elements, 58720256 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 +Params for bucket 18 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 +Params for bucket 20 (58720256 elements, 58720256 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 +Params for bucket 21 (58720256 elements, 58720256 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 +Params for bucket 23 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 +Params for bucket 24 (58720256 elements, 58720256 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 +Params for bucket 27 (58720256 elements, 58720256 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 +Params for bucket 28 (58720256 elements, 58720256 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 29 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 30 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 31 (58720256 elements, 58720256 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 +Params for bucket 34 (58720256 elements, 58720256 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 +Params for bucket 35 (58720256 elements, 58720256 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 +Params for bucket 36 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 +Params for bucket 37 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 38 (58720256 elements, 58720256 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 +Params for bucket 41 (58720256 elements, 58720256 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 +Params for bucket 42 (58720256 elements, 58720256 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 43 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 +Params for bucket 44 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 +Params for bucket 45 (58720256 elements, 58720256 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 +Params for bucket 47 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 +Params for bucket 48 (58720256 elements, 58720256 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 +Params for bucket 49 (58720256 elements, 58720256 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 50 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 52 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +Params for bucket 53 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 +Params for bucket 54 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 +Params for bucket 55 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 +Params for bucket 56 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 05:12:26 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 192 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(192, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 44ed1ee5bcb9e5b030387b8d27a6c406-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 05:12:26 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3466.83, 3478.95) + train/valid/test-data-iterators-setup ..........: (75.31, 91.23) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-30 05:12:26 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/capacity_attempts.tsv b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/capacity_attempts.tsv new file mode 100644 index 0000000000000000000000000000000000000000..521ff829b84d8b20fea1f26ede308016fee6a30d --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/capacity_attempts.tsv @@ -0,0 +1,5 @@ +model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir +G1 ultraep measure 1.25 OK 108731 0 0.6209174791971842 1.724609 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1 +G1 ultraep validate 1.8125 OK 113767 0 0.032329559326171875 2.309082 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1 +G2 ultraep measure 1.25 OK 110343 1 0.0 1.010101 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1 +G2 ultraep validate 1.0625 OK 107987 1 0.0 1.010101 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1 diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/capacity_selection.tsv b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/capacity_selection.tsv new file mode 100644 index 0000000000000000000000000000000000000000..6f65ef9c3cf1309f19e8cc4d1196b93d38a9d31f --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/capacity_selection.tsv @@ -0,0 +1,2 @@ +model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir +G1 ultraep ROUNDED_CAPACITY_VALIDATION_DROPPED diff --git a/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/results.tsv b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/results.tsv new file mode 100644 index 0000000000000000000000000000000000000000..8456066104d1b0840c5e981771e4992428e14220 --- /dev/null +++ b/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/results.tsv @@ -0,0 +1,5 @@ +config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir +G1 ultraep matched 1 4096 16 1.25 none 1 OK 893.49 886.90 108731 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p25/rep1 +G1 ultraep matched 1 4096 16 1.8125 none 1 OK 914.84 903.60 113767 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G1__ultraep__matched__seq4096__gbs16__cap1p8125/rep1 +G2 ultraep matched 4 4096 16 1.25 none 1 OK 558.91 532.70 110343 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p25/rep1 +G2 ultraep matched 4 4096 16 1.0625 none 1 OK 551.94 527.00 107987 /workspace/tiera-rerun-fixed-moonep/ultraep/tierbprobe/tierbprobe_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p0625/rep1